damien/FindMyShitV2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update fileSystem.ts

Browse Source
This commit is contained in:
Damien 2025-02-10 13:12:12 -05:00
parent 29f76c9e9d
commit 458ea75bb6
1 changed files with 69 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

105
electron-file-search/electron/services/fileSystem.ts
View File

@ -100,55 +100,88 @@ class FileSystemService {
const indexName = this.slugify(dirPath); const indexName = this.slugify(dirPath);
watcher.on('add', async (filePath) => { // Queue for files to be added to Meilisearch
console.log(`File ${filePath} has been added`); const fileQueue: string[] = [];
const MAX_QUEUE_SIZE = 1000;
let isProcessingQueue = false;
try { const processFileQueue = async () => {
const stats = await fs.promises.stat(filePath); if (isProcessingQueue) return;
const slug = this.slugify(filePath); isProcessingQueue = true;
const fileContent = await fs.promises.readFile(filePath, 'utf-8');
const fileExtension = path.extname(filePath);
const fileName = path.basename(filePath);
const permissions = { while (fileQueue.length > 0) {
read: !!(stats.mode & fs.constants.S_IRUSR), const batch = fileQueue.splice(0, 100); // Get the first 100 files
write: !!(stats.mode & fs.constants.S_IWUSR), const documents = [];
execute: !!(stats.mode & fs.constants.S_IXUSR),
};
const document = { for (const filePath of batch) {
id: slug, try {
name: filePath, const stats = await fs.promises.stat(filePath);
fileName: fileName, const slug = this.slugify(filePath);
content: fileContent, const fileContent = await fs.promises.readFile(filePath, 'utf-8');
extension: fileExtension, const fileExtension = path.extname(filePath);
createdAt: stats.birthtime, const fileName = path.basename(filePath);
modifiedAt: stats.mtime,
accessedAt: stats.atime,
size: stats.size,
permissions: permissions,
};
let fileHash: string | undefined; const permissions = {
try { read: !!(stats.mode & fs.constants.S_IRUSR),
fileHash = await this.calculateFileHash(filePath); write: !!(stats.mode & fs.constants.S_IWUSR),
document['hash'] = fileHash; execute: !!(stats.mode & fs.constants.S_IXUSR),
} catch (hashError) { };
console.error(`Failed to calculate file hash for ${filePath}:`, hashError);
const document = {
id: slug,
name: filePath,
fileName: fileName,
content: fileContent,
extension: fileExtension,
createdAt: stats.birthtime,
modifiedAt: stats.mtime,
accessedAt: stats.atime,
size: stats.size,
permissions: permissions,
};
let fileHash: string | undefined;
try {
fileHash = await this.calculateFileHash(filePath);
document['hash'] = fileHash;
} catch (hashError) {
console.error(`Failed to calculate file hash for ${filePath}:`, hashError);
}
documents.push(document);
} catch (error) {
console.error(`Failed to process file ${filePath}:`, error);
}
} }
if (this.meilisearchService) { if (this.meilisearchService) {
try { try {
await this.meilisearchService.addDocuments(indexName, [document]); await this.meilisearchService.addDocuments(indexName, documents);
console.log(`Added document to Meilisearch: ${slug}`); console.log(`Added ${documents.length} documents to Meilisearch`);
} catch (meilisearchError) { } catch (meilisearchError) {
console.error(`Failed to add document to Meilisearch:`, meilisearchError); console.error(`Failed to add documents to Meilisearch:`, meilisearchError);
} }
} else { } else {
console.warn('Meilisearch service not initialized.'); console.warn('Meilisearch service not initialized.');
} }
} catch (error) {
console.error(`Failed to add document to Meilisearch:`, error); // Wait before processing the next batch
await new Promise(resolve => setTimeout(resolve, 1000)); // 1 second delay
}
isProcessingQueue = false;
};
watcher.on('add', async (filePath) => {
console.log(`File ${filePath} has been added`);
if (fileQueue.length >= MAX_QUEUE_SIZE) {
console.log(`File queue is full. Skipping ${filePath}`);
return;
}
fileQueue.push(filePath);
if (!isProcessingQueue) {
processFileQueue();
} }
}); });