Update fileSystem.ts

electron-file-search/electron/services/fileSystem.ts

@@ -100,9 +100,20 @@ class FileSystemService {
 
     const indexName = this.slugify(dirPath);
 
-    watcher.on('add', async (filePath) => {
-      console.log(`File ${filePath} has been added`);
+    // Queue for files to be added to Meilisearch
+    const fileQueue: string[] = [];
+    const MAX_QUEUE_SIZE = 1000;
+    let isProcessingQueue = false;
 
+    const processFileQueue = async () => {
+      if (isProcessingQueue) return;
+      isProcessingQueue = true;
+
+      while (fileQueue.length > 0) {
+        const batch = fileQueue.splice(0, 100); // Get the first 100 files
+        const documents = [];
+
+        for (const filePath of batch) {
           try {
             const stats = await fs.promises.stat(filePath);
             const slug = this.slugify(filePath);
@@ -136,19 +147,41 @@ class FileSystemService {
             } catch (hashError) {
               console.error(`Failed to calculate file hash for ${filePath}:`, hashError);
             }
+            documents.push(document);
+          } catch (error) {
+            console.error(`Failed to process file ${filePath}:`, error);
+          }
+        }
 
         if (this.meilisearchService) {
           try {
-            await this.meilisearchService.addDocuments(indexName, [document]);
-            console.log(`Added document to Meilisearch: ${slug}`);
+            await this.meilisearchService.addDocuments(indexName, documents);
+            console.log(`Added ${documents.length} documents to Meilisearch`);
           } catch (meilisearchError) {
-            console.error(`Failed to add document to Meilisearch:`, meilisearchError);
+            console.error(`Failed to add documents to Meilisearch:`, meilisearchError);
           }
         } else {
           console.warn('Meilisearch service not initialized.');
         }
-      } catch (error) {
-        console.error(`Failed to add document to Meilisearch:`, error);
+
+        // Wait before processing the next batch
+        await new Promise(resolve => setTimeout(resolve, 1000)); // 1 second delay
+      }
+
+      isProcessingQueue = false;
+    };
+
+    watcher.on('add', async (filePath) => {
+      console.log(`File ${filePath} has been added`);
+
+      if (fileQueue.length >= MAX_QUEUE_SIZE) {
+        console.log(`File queue is full. Skipping ${filePath}`);
+        return;
+      }
+
+      fileQueue.push(filePath);
+      if (!isProcessingQueue) {
+        processFileQueue();
       }
     });