import { FSWatcher } from 'chokidar'; import * as chokidar from 'chokidar'; import Store from 'electron-store'; import { ServiceError } from '../types'; import * as path from 'path'; const fs = require('fs'); const fsPromises = fs.promises; import * as crypto from 'crypto'; import MeilisearchService from './meilisearchService'; const store = new Store<{ watchedPaths: string[]; excludedPaths: string[]; }>(); class FileSystemService { private watchers: Map; private excludedPaths: Set; private meilisearchService: MeilisearchService; private readonly indexName = 'files'; constructor() { this.watchers = new Map(); this.excludedPaths = new Set(store.get('excludedPaths', [])); this.meilisearchService = new MeilisearchService(); // Add example paths const examplePaths = [ 'C:/example/llm-only', 'C:/example/ocr-only', 'C:/example/both-enabled' ]; // Set initial watched paths store.set('watchedPaths', examplePaths); // Start watching example paths examplePaths.forEach(path => { this.watchers.set(path, null); // Add to watchers without actual watcher since paths don't exist }); } private slugify(filePath: string): string { return filePath .replace(/[\\/]/g, '-') // Replace path separators with dashes .replace(/[^a-zA-Z0-9_-]+/g, '') // Remove non-alphanumeric characters .toLowerCase(); } private async calculateFileHash(filePath: string): Promise { return new Promise((resolve, reject) => { try { const hash = crypto.createHash('sha256'); const stream = fs.createReadStream(filePath); if (!stream) { reject(new Error(`Failed to create read stream for ${filePath}`)); return; } stream.on('data', (data: any) => { try { hash.update(data); } catch (dataError) { reject(new Error(`Failed to update hash with data for ${filePath}: ${dataError}`)); } }); stream.on('end', () => { try { resolve(hash.digest('hex')); } catch (digestError) { reject(new Error(`Failed to digest hash for ${filePath}: ${digestError}`)); } }); stream.on('error', (streamError: any) => { reject(new Error(`Read stream error for ${filePath}: ${streamError}`)); }); } catch (creationError: any) { reject(new Error(`Failed to create read stream or hash for ${filePath}: ${creationError}`)); } }); } public async startWatching(dirPath: string): Promise { if (this.watchers.has(dirPath)) { throw new ServiceError(`Already watching directory: ${dirPath}`); } const watcher = chokidar.watch(dirPath, { ignored: [ /(^|[\\/])\../, // Ignore dotfiles '**/node_modules/**', ...Array.from(this.excludedPaths), ], persistent: true, ignoreInitial: false, }); const indexName = this.slugify(dirPath); // Queue for files to be added to Meilisearch const fileQueue: string[] = []; const MAX_QUEUE_SIZE = 1000; let isProcessingQueue = false; const processFileQueue = async () => { if (isProcessingQueue) return; isProcessingQueue = true; while (fileQueue.length > 0) { const batch = fileQueue.splice(0, 100); // Get the first 100 files const documents = []; for (const filePath of batch) { try { const stats = await fs.promises.stat(filePath); const slug = this.slugify(filePath); const fileContent = await fs.promises.readFile(filePath, 'utf-8'); const fileExtension = path.extname(filePath); const fileName = path.basename(filePath); const permissions = { read: !!(stats.mode & fs.constants.S_IRUSR), write: !!(stats.mode & fs.constants.S_IWUSR), execute: !!(stats.mode & fs.constants.S_IXUSR), }; const document = { id: slug, name: filePath, fileName: fileName, content: fileContent, extension: fileExtension, createdAt: stats.birthtime, modifiedAt: stats.mtime, accessedAt: stats.atime, size: stats.size, permissions: permissions, }; let fileHash: string | undefined; try { fileHash = await this.calculateFileHash(filePath); document['hash'] = fileHash; } catch (hashError) { console.error(`Failed to calculate file hash for ${filePath}:`, hashError); } documents.push(document); } catch (error) { console.error(`Failed to process file ${filePath}:`, error); } } if (this.meilisearchService) { try { await this.meilisearchService.addDocuments(indexName, documents); console.log(`Added ${documents.length} documents to Meilisearch`); } catch (meilisearchError) { console.error(`Failed to add documents to Meilisearch:`, meilisearchError); } } else { console.warn('Meilisearch service not initialized.'); } // Wait before processing the next batch await new Promise(resolve => setTimeout(resolve, 1000)); // 1 second delay } isProcessingQueue = false; }; watcher.on('add', async (filePath) => { console.log(`File ${filePath} has been added`); if (fileQueue.length >= MAX_QUEUE_SIZE) { console.log(`File queue is full. Skipping ${filePath}`); return; } fileQueue.push(filePath); if (!isProcessingQueue) { processFileQueue(); } }); watcher.on('change', path => { console.log(`File ${path} has been changed`); // TODO: Process file changes }); watcher.on('unlink', path => { console.log(`File ${path} has been removed`); // TODO: Remove from vector store }); this.watchers.set(dirPath, watcher); const watchedPaths = store.get('watchedPaths', []); if (!watchedPaths.includes(dirPath)) { store.set('watchedPaths', [...watchedPaths, dirPath]); } } public async stopWatching(dirPath: string): Promise { const watcher = this.watchers.get(dirPath); if (!watcher) { throw new ServiceError(`Not watching directory: ${dirPath}`); } await watcher.close(); this.watchers.delete(dirPath); const watchedPaths = store.get('watchedPaths', []); store.set('watchedPaths', watchedPaths.filter(p => p !== dirPath)); } public addExcludedPath(path: string): void { this.excludedPaths.add(path); store.set('excludedPaths', Array.from(this.excludedPaths)); // Update all watchers with new excluded path this.watchers.forEach(watcher => { watcher.unwatch(path); }); } public getWatchedPaths(): string[] { return Array.from(this.watchers.keys()); } public getExcludedPaths(): string[] { return Array.from(this.excludedPaths); } } export const fileSystemService = new FileSystemService();