Update fileSystem.ts

electron-file-search/electron/services/fileSystem.ts

@@ -100,55 +100,88 @@ class FileSystemService {
 
     const indexName = this.slugify(dirPath);
 
-    watcher.on('add', async (filePath) => {
-      console.log(`File ${filePath} has been added`);
+    // Queue for files to be added to Meilisearch
+    const fileQueue: string[] = [];
+    const MAX_QUEUE_SIZE = 1000;
+    let isProcessingQueue = false;
 
-      try {
-        const stats = await fs.promises.stat(filePath);
-        const slug = this.slugify(filePath);
-        const fileContent = await fs.promises.readFile(filePath, 'utf-8');
-        const fileExtension = path.extname(filePath);
-        const fileName = path.basename(filePath);
+    const processFileQueue = async () => {
+      if (isProcessingQueue) return;
+      isProcessingQueue = true;
 
-        const permissions = {
-          read: !!(stats.mode & fs.constants.S_IRUSR),
-          write: !!(stats.mode & fs.constants.S_IWUSR),
-          execute: !!(stats.mode & fs.constants.S_IXUSR),
-        };
+      while (fileQueue.length > 0) {
+        const batch = fileQueue.splice(0, 100); // Get the first 100 files
+        const documents = [];
 
-        const document = {
-          id: slug,
-          name: filePath,
-          fileName: fileName,
-          content: fileContent,
-          extension: fileExtension,
-          createdAt: stats.birthtime,
-          modifiedAt: stats.mtime,
-          accessedAt: stats.atime,
-          size: stats.size,
-          permissions: permissions,
-        };
+        for (const filePath of batch) {
+          try {
+            const stats = await fs.promises.stat(filePath);
+            const slug = this.slugify(filePath);
+            const fileContent = await fs.promises.readFile(filePath, 'utf-8');
+            const fileExtension = path.extname(filePath);
+            const fileName = path.basename(filePath);
 
-        let fileHash: string | undefined;
-        try {
-          fileHash = await this.calculateFileHash(filePath);
-          document['hash'] = fileHash;
-        } catch (hashError) {
-          console.error(`Failed to calculate file hash for ${filePath}:`, hashError);
+            const permissions = {
+              read: !!(stats.mode & fs.constants.S_IRUSR),
+              write: !!(stats.mode & fs.constants.S_IWUSR),
+              execute: !!(stats.mode & fs.constants.S_IXUSR),
+            };
+
+            const document = {
+              id: slug,
+              name: filePath,
+              fileName: fileName,
+              content: fileContent,
+              extension: fileExtension,
+              createdAt: stats.birthtime,
+              modifiedAt: stats.mtime,
+              accessedAt: stats.atime,
+              size: stats.size,
+              permissions: permissions,
+            };
+
+            let fileHash: string | undefined;
+            try {
+              fileHash = await this.calculateFileHash(filePath);
+              document['hash'] = fileHash;
+            } catch (hashError) {
+              console.error(`Failed to calculate file hash for ${filePath}:`, hashError);
+            }
+            documents.push(document);
+          } catch (error) {
+            console.error(`Failed to process file ${filePath}:`, error);
+          }
         }
 
         if (this.meilisearchService) {
           try {
-            await this.meilisearchService.addDocuments(indexName, [document]);
-            console.log(`Added document to Meilisearch: ${slug}`);
+            await this.meilisearchService.addDocuments(indexName, documents);
+            console.log(`Added ${documents.length} documents to Meilisearch`);
           } catch (meilisearchError) {
-            console.error(`Failed to add document to Meilisearch:`, meilisearchError);
+            console.error(`Failed to add documents to Meilisearch:`, meilisearchError);
           }
         } else {
           console.warn('Meilisearch service not initialized.');
         }
-      } catch (error) {
-        console.error(`Failed to add document to Meilisearch:`, error);
+
+        // Wait before processing the next batch
+        await new Promise(resolve => setTimeout(resolve, 1000)); // 1 second delay
+      }
+
+      isProcessingQueue = false;
+    };
+
+    watcher.on('add', async (filePath) => {
+      console.log(`File ${filePath} has been added`);
+
+      if (fileQueue.length >= MAX_QUEUE_SIZE) {
+        console.log(`File queue is full. Skipping ${filePath}`);
+        return;
+      }
+
+      fileQueue.push(filePath);
+      if (!isProcessingQueue) {
+        processFileQueue();
       }
     });