FindMyShitV2/electron-file-search/electron/services/fileSystem.ts

import { FSWatcher } from 'chokidar';
import * as chokidar from 'chokidar';
import Store from 'electron-store';
import { ServiceError } from '../types';
import * as path from 'path';
const fs = require('fs');
const fsPromises = fs.promises;
import * as crypto from 'crypto';
import MeilisearchService from './meilisearchService';

const store = new Store<{
  watchedPaths: string[];
  excludedPaths: string[];
}>();

class FileSystemService {
  private watchers: Map<string, FSWatcher>;
  private excludedPaths: Set<string>;
  private meilisearchService: MeilisearchService;
  private readonly indexName = 'files';

  constructor() {
    this.watchers = new Map();
    this.excludedPaths = new Set(store.get('excludedPaths', []));
    this.meilisearchService = new MeilisearchService();

    // Add example paths
    const examplePaths = [
      'C:/example/llm-only',
      'C:/example/ocr-only',
      'C:/example/both-enabled'
    ];

    // Set initial watched paths
    store.set('watchedPaths', examplePaths);

    // Start watching example paths
    examplePaths.forEach(path => {
      this.watchers.set(path, null); // Add to watchers without actual watcher since paths don't exist
    });
  }

  private slugify(filePath: string): string {
    return filePath
      .replace(/[\\/]/g, '-') // Replace path separators with dashes
      .replace(/[^a-zA-Z0-9_-]+/g, '') // Remove non-alphanumeric characters
      .toLowerCase();
  }

  private async calculateFileHash(filePath: string): Promise<string> {
    return new Promise((resolve, reject) => {
      try {
        const hash = crypto.createHash('sha256');
        const stream = fs.createReadStream(filePath);

        if (!stream) {
          reject(new Error(`Failed to create read stream for ${filePath}`));
          return;
        }

        stream.on('data', (data: any) => {
          try {
            hash.update(data);
          } catch (dataError) {
            reject(new Error(`Failed to update hash with data for ${filePath}: ${dataError}`));
          }
        });

        stream.on('end', () => {
          try {
            resolve(hash.digest('hex'));
          } catch (digestError) {
            reject(new Error(`Failed to digest hash for ${filePath}: ${digestError}`));
          }
        });

        stream.on('error', (streamError: any) => {
          reject(new Error(`Read stream error for ${filePath}: ${streamError}`));
        });
      } catch (creationError: any) {
        reject(new Error(`Failed to create read stream or hash for ${filePath}: ${creationError}`));
      }
    });
  }

  public async startWatching(dirPath: string): Promise<void> {
    if (this.watchers.has(dirPath)) {
      throw new ServiceError(`Already watching directory: ${dirPath}`);
    }

    const watcher = chokidar.watch(dirPath, {
      ignored: [
        /(^|[\\/])\../, // Ignore dotfiles
        '**/node_modules/**',
        ...Array.from(this.excludedPaths),
      ],
      persistent: true,
      ignoreInitial: false,
    });

    const indexName = this.slugify(dirPath);

    // Queue for files to be added to Meilisearch
    const fileQueue: string[] = [];
    const MAX_QUEUE_SIZE = 1000;
    let isProcessingQueue = false;

    const processFileQueue = async () => {
      if (isProcessingQueue) return;
      isProcessingQueue = true;

      while (fileQueue.length > 0) {
        const batch = fileQueue.splice(0, 100); // Get the first 100 files
        const documents = [];

        for (const filePath of batch) {
          try {
            const stats = await fs.promises.stat(filePath);
            const slug = this.slugify(filePath);
            const fileContent = await fs.promises.readFile(filePath, 'utf-8');
            const fileExtension = path.extname(filePath);
            const fileName = path.basename(filePath);

            const permissions = {
              read: !!(stats.mode & fs.constants.S_IRUSR),
              write: !!(stats.mode & fs.constants.S_IWUSR),
              execute: !!(stats.mode & fs.constants.S_IXUSR),
            };

            const document = {
              id: slug,
              name: filePath,
              fileName: fileName,
              content: fileContent,
              extension: fileExtension,
              createdAt: stats.birthtime,
              modifiedAt: stats.mtime,
              accessedAt: stats.atime,
              size: stats.size,
              permissions: permissions,
            };

            let fileHash: string | undefined;
            try {
              fileHash = await this.calculateFileHash(filePath);
              document['hash'] = fileHash;
            } catch (hashError) {
              console.error(`Failed to calculate file hash for ${filePath}:`, hashError);
            }
            documents.push(document);
          } catch (error) {
            console.error(`Failed to process file ${filePath}:`, error);
          }
        }

        if (this.meilisearchService) {
          try {
            await this.meilisearchService.addDocuments(indexName, documents);
            console.log(`Added ${documents.length} documents to Meilisearch`);
          } catch (meilisearchError) {
            console.error(`Failed to add documents to Meilisearch:`, meilisearchError);
          }
        } else {
          console.warn('Meilisearch service not initialized.');
        }

        // Wait before processing the next batch
        await new Promise(resolve => setTimeout(resolve, 1000)); // 1 second delay
      }

      isProcessingQueue = false;
    };

    watcher.on('add', async (filePath) => {
      console.log(`File ${filePath} has been added`);

      if (fileQueue.length >= MAX_QUEUE_SIZE) {
        console.log(`File queue is full. Skipping ${filePath}`);
        return;
      }

      fileQueue.push(filePath);
      if (!isProcessingQueue) {
        processFileQueue();
      }
    });

    watcher.on('change', path => {
      console.log(`File ${path} has been changed`);
      // TODO: Process file changes
    });

    watcher.on('unlink', path => {
      console.log(`File ${path} has been removed`);
      // TODO: Remove from vector store
    });

    this.watchers.set(dirPath, watcher);
    const watchedPaths = store.get('watchedPaths', []);
    if (!watchedPaths.includes(dirPath)) {
      store.set('watchedPaths', [...watchedPaths, dirPath]);
    }
  }

  public async stopWatching(dirPath: string): Promise<void> {
    const watcher = this.watchers.get(dirPath);
    if (!watcher) {
      throw new ServiceError(`Not watching directory: ${dirPath}`);
    }

    await watcher.close();
    this.watchers.delete(dirPath);

    const watchedPaths = store.get('watchedPaths', []);
    store.set('watchedPaths', watchedPaths.filter(p => p !== dirPath));
  }

  public addExcludedPath(path: string): void {
    this.excludedPaths.add(path);
    store.set('excludedPaths', Array.from(this.excludedPaths));

    // Update all watchers with new excluded path
    this.watchers.forEach(watcher => {
      watcher.unwatch(path);
    });
  }

  public getWatchedPaths(): string[] {
    return Array.from(this.watchers.keys());
  }

  public getExcludedPaths(): string[] {
    return Array.from(this.excludedPaths);
  }
}

export const fileSystemService = new FileSystemService();