Merge remote-tracking branch 'origin/dev' into dev

backend/src/modules/document-import/document-import.service.ts

@@ -33,6 +33,8 @@ type ImportSourceConnection = {
 }
 
 const activeSyncs = new Set<string>()
+const syncProgress = new Map<string, { processed: number, total: number, imported: number, duplicates: number }>()
+const IMPORT_BATCH_SIZE = 25
 
 const decryptString = (value: unknown) => value ? decrypt(value as any) : ""
 
@@ -215,17 +217,18 @@ export function documentImportService(server: FastifyInstance) {
 
     const syncSource = async (tenantId: number, sourceId: string) => {
         if (activeSyncs.has(sourceId)) throw new Error("Diese Importquelle wird bereits synchronisiert")
+        activeSyncs.add(sourceId)
+        let client: ImapFlow | null = null
+        let imported = 0
+        let duplicates = 0
+        let messages = 0
+        try {
             const source = await loadSource(tenantId, sourceId)
             if (!source) throw new Error("Importquelle wurde nicht gefunden")
             if (!source.enabled) throw new Error("Importquelle ist deaktiviert")
             if (source.provider !== "imap") throw new Error("Dieser Provider wird noch nicht unterstützt")
 
-        activeSyncs.add(sourceId)
+            client = createImapClient(source)
-        const client = createImapClient(source)
-        let imported = 0
-        let duplicates = 0
-        let messages = 0
-        try {
             await client.connect()
             const lock = await client.getMailboxLock(source.mailboxPath)
             try {
@@ -237,11 +240,19 @@ export function documentImportService(server: FastifyInstance) {
                 )).limit(1)
                 const highestUid = state && Number(state.uidValidity) === uidValidity ? Number(state.highestUid) : 0
                 let processedHighestUid = highestUid
-                // Ungelesene Nachrichten werden immer berücksichtigt. So bleibt ein fehlgeschlagener
+                // Pro Lauf nur einen begrenzten Stapel laden. Der Worker verarbeitet verbliebene
-                // Import erneut verarbeitbar, auch wenn danach bereits neuere UIDs erfolgreich waren.
+                // ungelesene Nachrichten in den folgenden Läufen weiter.
-                const query: any = { seen: false }
+                const unreadUids = await client.search({ seen: false }, { uid: true }) || []
+                const batchUids = unreadUids.slice(0, IMPORT_BATCH_SIZE)
+                syncProgress.set(sourceId, {
+                    processed: 0,
+                    total: batchUids.length,
+                    imported: 0,
+                    duplicates: 0,
+                })
+                const uidsToMarkSeen: number[] = []
 
-                for await (const message of client.fetch(query, { uid: true, envelope: true, source: true })) {
+                for await (const message of client.fetch(batchUids, { uid: true, envelope: true, source: true }, { uid: true })) {
                     messages += 1
                     const parsed = await simpleParser(message.source)
                     const remoteMessageId = `${uidValidity}:${message.uid}`
@@ -256,8 +267,19 @@ export function documentImportService(server: FastifyInstance) {
                             complete = false
                         }
                     }
-                    if (complete && source.markAsSeen) await client.messageFlagsAdd({ uid: message.uid }, ["\\Seen"], { uid: true })
+                    // Während eines laufenden fetch dürfen keine weiteren IMAP-Befehle ausgeführt
+                    // werden, da ImapFlow sonst auf das Ende des eigenen fetch wartet.
+                    if (complete && source.markAsSeen) uidsToMarkSeen.push(message.uid)
                     if (complete) processedHighestUid = Math.max(processedHighestUid, Number(message.uid))
+                    syncProgress.set(sourceId, {
+                        processed: messages,
+                        total: batchUids.length,
+                        imported,
+                        duplicates,
+                    })
+                }
+                if (uidsToMarkSeen.length > 0) {
+                    await client.messageFlagsAdd(uidsToMarkSeen, ["\\Seen"], { uid: true })
                 }
 
                 await server.db.insert(documentImportStates).values({
@@ -277,14 +299,28 @@ export function documentImportService(server: FastifyInstance) {
             await server.db.update(documentImportSources).set({ lastSyncedAt: new Date(), lastError: null }).where(eq(documentImportSources.id, source.id))
             return { success: true, messages, imported, duplicates }
         } catch (error: any) {
-            await server.db.update(documentImportSources).set({ lastError: error?.message || "Synchronisierung fehlgeschlagen" }).where(eq(documentImportSources.id, source.id))
+            await server.db.update(documentImportSources).set({ lastError: error?.message || "Synchronisierung fehlgeschlagen" }).where(eq(documentImportSources.id, sourceId))
             throw error
         } finally {
             activeSyncs.delete(sourceId)
-            if (client.usable) await client.logout().catch(() => client.close())
+            syncProgress.delete(sourceId)
+            if (client?.usable) await client.logout().catch(() => client.close())
         }
     }
 
+    const startSync = (tenantId: number, sourceId: string) => {
+        if (activeSyncs.has(sourceId)) throw new Error("Diese Importquelle wird bereits synchronisiert")
+        void syncSource(tenantId, sourceId).catch((error: any) => {
+            server.log.error({ sourceId, error: error?.message }, "Manueller Dokumentenimport fehlgeschlagen")
+        })
+        return { success: true, started: true }
+    }
+
+    const getSyncStatus = (sourceId: string) => ({
+        isSyncing: activeSyncs.has(sourceId),
+        progress: syncProgress.get(sourceId) || null,
+    })
+
     const syncAll = async () => {
         const sources = await server.db.select({
             id: documentImportSources.id,
@@ -303,5 +339,5 @@ export function documentImportService(server: FastifyInstance) {
         return results
     }
 
-    return { testConnection, syncSource, syncAll }
+    return { testConnection, syncSource, startSync, getSyncStatus, syncAll }
 }

backend/src/routes/documentImports.ts

@@ -29,6 +29,7 @@ export default async function documentImportRoutes(server: FastifyInstance) {
         lastSyncedAt: row.lastSyncedAt,
         lastError: row.lastError,
         createdAt: row.createdAt,
+        ...server.services.documentImports.getSyncStatus(row.id),
     })
 
     server.get("/document-imports", async (req) => {
@@ -101,7 +102,10 @@ export default async function documentImportRoutes(server: FastifyInstance) {
 
     server.post("/document-imports/:id/sync", async (req, reply) => {
         try {
-            return await server.services.documentImports.syncSource(tenantId(req), (req.params as { id: string }).id)
+            return reply.code(202).send(server.services.documentImports.startSync(
+                tenantId(req),
+                (req.params as { id: string }).id,
+            ))
         } catch (error: any) {
             return reply.code(400).send({ error: error?.message || "Synchronisierung fehlgeschlagen" })
         }

frontend/pages/settings/document-imports/index.vue

@@ -13,6 +13,13 @@ type ImportSource = {
   hasPassword: boolean
   lastSyncedAt?: string | null
   lastError?: string | null
+  isSyncing: boolean
+  progress?: {
+    processed: number
+    total: number
+    imported: number
+    duplicates: number
+  } | null
 }
 
 const api = useNuxtApp().$api
@@ -23,6 +30,7 @@ const saving = ref(false)
 const activeAction = ref<string | null>(null)
 const editingId = ref<string | null>(null)
 const showForm = ref(false)
+let pollTimer: ReturnType<typeof setTimeout> | null = null
 
 const emptyForm = () => ({
   name: "",
@@ -38,23 +46,33 @@ const emptyForm = () => ({
 })
 const form = ref(emptyForm())
 
-const load = async () => {
+const load = async (silent = false) => {
-  loading.value = true
+  loading.value = sources.value.length === 0
   try {
     const result = await api<ImportSource[]>("/api/document-imports")
     sources.value = Array.isArray(result) ? result : []
   } catch (error: any) {
-    sources.value = []
+    if (!silent) {
       toast.add({
         title: "Importquellen konnten nicht geladen werden",
         description: error?.data?.error || error?.message || "Bitte versuche es später erneut.",
         color: "error",
       })
+    }
   } finally {
     loading.value = false
   }
 }
 
+const pollWhileSyncing = () => {
+  if (pollTimer || !sources.value.some(source => source.isSyncing)) return
+  pollTimer = setTimeout(async () => {
+    pollTimer = null
+    await load(true)
+    pollWhileSyncing()
+  }, 2000)
+}
+
 const create = () => {
   editingId.value = null
   form.value = emptyForm()
@@ -100,6 +118,13 @@ const runAction = async (id: string, action: "test" | "sync") => {
   activeAction.value = `${id}:${action}`
   try {
     const result = await api(`/api/document-imports/${id}/${action}`, { method: "POST" })
+    if (action === "sync" && result.started) {
+      const source = sources.value.find(item => item.id === id)
+      if (source) source.isSyncing = true
+      toast.add({ title: "Import gestartet", description: "Die Nachrichten werden im Hintergrund verarbeitet.", color: "success" })
+      pollWhileSyncing()
+      return
+    }
     toast.add({
       title: action === "test" ? "Verbindung erfolgreich" : "Import abgeschlossen",
       description: action === "test"
@@ -120,7 +145,13 @@ const runAction = async (id: string, action: "test" | "sync") => {
   }
 }
 
-onMounted(load)
+onMounted(async () => {
+  await load()
+  pollWhileSyncing()
+})
+onBeforeUnmount(() => {
+  if (pollTimer) clearTimeout(pollTimer)
+})
 </script>
 
 <template>
@@ -160,6 +191,9 @@ onMounted(load)
             <p v-if="source.lastSyncedAt" class="mt-1 text-xs text-dimmed">
               Zuletzt synchronisiert: {{ new Date(source.lastSyncedAt).toLocaleString('de-DE') }}
             </p>
+            <p v-if="source.isSyncing" class="mt-1 text-sm text-primary">
+              Import läuft<span v-if="source.progress">: {{ source.progress.processed }} von {{ source.progress.total }} Nachrichten verarbeitet</span>
+            </p>
             <p v-if="source.lastError" class="mt-1 text-sm text-error">{{ source.lastError }}</p>
           </div>
           <div class="flex shrink-0 flex-wrap gap-2">
@@ -172,8 +206,8 @@ onMounted(load)
             >Testen</UButton>
             <UButton
               icon="i-heroicons-arrow-path"
-              :loading="activeAction === `${source.id}:sync`"
+              :loading="activeAction === `${source.id}:sync` || source.isSyncing"
-              :disabled="!source.enabled"
+              :disabled="!source.enabled || source.isSyncing"
               @click="runAction(source.id, 'sync')"
             >Jetzt importieren</UButton>
             <UButton color="neutral" variant="ghost" icon="i-heroicons-pencil-square" @click="edit(source)" />