damien/meilisearch.NET
1
0
Fork 0
Code Issues Actions Packages Projects Releases 16 Wiki Activity

feat: meilisearch index enable/disable automatic compression + usage method

Browse Source
This commit is contained in:
Damien 2025-03-01 03:13:00 -05:00
parent e185f2925e
commit 99bf0017c1
7 changed files with 361 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

82
meilisearch.NET.example/Program.cs
View File

@ -62,42 +62,52 @@ public class test
Task.Delay(1000).Wait(); // Wait for 1 second before checking again
}
service.CreateIndex<document>("test");
service.AddDocument("test", new document()
{
Id = Guid.NewGuid(),
message = "Hello, Meilisearch!"
});
service.AddDocument("test", new document()
{
Id = Guid.NewGuid(),
message = "Hello, Meilisearch!"
});
service.AddDocument("test", new document()
{
Id = Guid.NewGuid(),
message = "Hello, Meilisearch!"
});
service.AddDocument("test", new document()
{
Id = Guid.NewGuid(),
message = "Hello, Meilisearch!"
});
service.AddDocument("test", new document()
{
Id = Guid.NewGuid(),
message = "Hello, Meilisearch!"
});
service.AddDocument("test", new document()
{
Id = Guid.NewGuid(),
message = "Hello, Meilisearch!"
});
service.AddDocument("test", new document()
{
Id = Guid.NewGuid(),
message = "Hello, Meilisearch!"
});
var usage = service.GetProcessResourceUsage();
_logger.LogInformation($"Memory usage: {usage.MemoryUsageBytes} MB");
_logger.LogInformation($"CPU usage: {usage.CpuPercentage} %");
_logger.LogInformation($"Disk read: {usage.DiskReadBytes} MB");
_logger.LogInformation($"Disk write: {usage.DiskWriteBytes} MB");
_logger.LogInformation($"Thread count: {usage.ThreadCount}");
_logger.LogInformation($"Process ID: {usage.ProcessId}");
//service.UpdateIndexStatus("test",false).Wait();
// service.AddDocument("test", new document()
// {
// Id = Guid.NewGuid(),
// message = "Hello, Meilisearch!"
// });
// service.AddDocument("test", new document()
// {
// Id = Guid.NewGuid(),
// message = "Hello, Meilisearch!"
// });
// service.AddDocument("test", new document()
// {
// Id = Guid.NewGuid(),
// message = "Hello, Meilisearch!"
// });
// service.AddDocument("test", new document()
// {
// Id = Guid.NewGuid(),
// message = "Hello, Meilisearch!"
// });
// service.AddDocument("test", new document()
// {
// Id = Guid.NewGuid(),
// message = "Hello, Meilisearch!"
// });
// service.AddDocument("test", new document()
// {
// Id = Guid.NewGuid(),
// message = "Hello, Meilisearch!"
// });
// service.AddDocument("test", new document()
// {
// Id = Guid.NewGuid(),
// message = "Hello, Meilisearch!"
// });
_logger.LogInformation("Test service initialized.");
}
}

328
meilisearch.NET/MeiliSearchService.cs
View File

@ -1,6 +1,7 @@
using System.Collections.ObjectModel;
using System.Collections.Specialized;
using System.Diagnostics;
using System.IO.Compression;
using System.Net.Sockets;
using System.Reflection;
using System.Runtime.InteropServices;
@ -12,6 +13,8 @@ using meilisearch.NET.Configurations;
using meilisearch.NET.Enums;
using meilisearch.NET.Extensions;
using meilisearch.NET.Interfaces;
using meilisearch.NET.Models;
using Meilisearch.QueryParameters;
using Microsoft.Extensions.Logging;
using Index = Meilisearch.Index;
@ -20,15 +23,19 @@ namespace meilisearch.NET;
public class MeiliSearchService:IDisposable
{
public readonly HttpClient _httpClient;
public readonly ILogger<MeiliSearchService> _logger;
public readonly MeilisearchClient _client;
public readonly MeiliSearchConfiguration _meiliConfiguration;
public readonly string _indexBasePath = Path.Combine(AppContext.BaseDirectory, "db", "indexes" );
public static string _apiKey = GenerateApiKey();
public const int THRESHOLD = 10000;
public Process? process;
public ObservableCollection<KeyValuePair<string,IDocument>> _documentCollection;
private readonly HttpClient _httpClient;
private readonly ILogger<MeiliSearchService> _logger;
private readonly MeilisearchClient _client;
private readonly MeiliSearchConfiguration _meiliConfiguration;
private readonly string _indexBasePath = Path.Combine(AppContext.BaseDirectory, "db", "indexes" );
private static string _apiKey = GenerateApiKey();
private const int THRESHOLD = 10000;
private const string DEFAULT_DATA_FILE_PATH = "data.mdb";
private const string DEFAULT_LOCK_FILE_PATH = "lock.mdb";
private Process? process;
private ObservableCollection<KeyValuePair<string,IDocument>> _documentCollection;
private List<Index> indexes { get; set; } = new();
protected virtual ObservableCollection<KeyValuePair<string, IDocument>> DocumentCollection
{
get
@ -53,8 +60,135 @@ public class MeiliSearchService:IDisposable
#region Private
public static string GenerateApiKey(int length = 64)
#region Private
private async Task CompressIndex(string indexName)
{
var indexPath = await GetIndexFilePath(indexName);
if (!Directory.Exists(indexPath))
{
_logger.LogWarning($"Index directory not found at: {indexPath}");
return;
}
var compressedPath = indexPath + ".zip";
_logger.LogTrace($"Compressing index '{indexName}' to {compressedPath}...");
try
{
var size = new DirectoryInfo(indexPath).GetFiles().Sum(f => f.Length);
// Create temp directory to ensure we don't lose data if compression fails
var tempPath = compressedPath + ".temp";
ZipFile.CreateFromDirectory(indexPath, tempPath, CompressionLevel.SmallestSize, false);
// If compression succeeded, safely replace old zip (if exists) and remove original directory
if (File.Exists(compressedPath))
{
File.Delete(compressedPath);
}
File.Move(tempPath, compressedPath);
Directory.Delete(indexPath, true);
// Update index metadata
var indexBindings = await _client.GetIndexAsync("index_bindings");
var doc = await _client.GetIndexAsync("index_bindings").Result.GetDocumentAsync<Models.Index>(indexName);
var document = new Models.Index
{
Name = indexName,
IsCompressed = true,
FolderId = doc.FolderId,
CreatedAt = doc.CreatedAt,
SizeBeforeCompression = size,
LastCompressedAt = DateTime.UtcNow
};
await indexBindings.UpdateDocumentsAsync(new List<Models.Index> { document });
_logger.LogInformation($"Successfully compressed index '{indexName}'");
Directory.CreateDirectory(indexPath);
File.Copy(Path.Combine(AppContext.BaseDirectory,DEFAULT_DATA_FILE_PATH), Path.Combine(indexPath, DEFAULT_DATA_FILE_PATH));
File.Copy(Path.Combine(AppContext.BaseDirectory,DEFAULT_LOCK_FILE_PATH), Path.Combine(indexPath, DEFAULT_LOCK_FILE_PATH));
_logger.LogInformation($"Created placeholder data file for compressed index '{indexName}'");
}
catch (Exception ex)
{
_logger.LogError($"Failed to compress index '{indexName}': {ex.Message}");
throw;
}
Stop();
}
private async Task DecompressIndex(string indexName)
{
var compressedPath = await GetIndexFilePath(indexName) + ".zip";
var extractPath = await GetIndexFilePath(indexName);
if (!File.Exists(compressedPath))
{
_logger.LogWarning($"Compressed index not found at: {compressedPath}");
return;
}
_logger.LogTrace($"Decompressing index '{indexName}' to {extractPath}...");
try
{
if (Directory.Exists(extractPath))
{
Directory.Delete(extractPath, true);
}
// Create temp directory to ensure we don't lose data if decompression fails
var tempPath = extractPath + ".temp";
ZipFile.ExtractToDirectory(compressedPath, tempPath);
// If decompression succeeded, safely move to final location
if (Directory.Exists(extractPath))
{
Directory.Delete(extractPath, true);
}
Directory.Move(tempPath, extractPath);
File.Delete(compressedPath);
// Update index metadata
var indexBindings = await _client.GetIndexAsync("index_bindings");
var doc = await _client.GetIndexAsync("index_bindings").Result.GetDocumentAsync<Models.Index>(indexName);
var document = new Models.Index
{
Name = indexName,
FolderId = doc.FolderId,
CreatedAt = doc.CreatedAt,
SizeBeforeCompression = null,
IsCompressed = false,
LastCompressedAt = doc.LastCompressedAt
};
await indexBindings.UpdateDocumentsAsync(new List<Models.Index> { document });
_logger.LogInformation($"Successfully decompressed index '{indexName}'");
}
catch (Exception ex)
{
_logger.LogError($"Failed to decompress index '{indexName}': {ex.Message}");
throw;
}
}
private static string FormatBytes(long bytes)
{
string[] sizes = { "B", "KB", "MB", "GB", "TB" };
int order = 0;
double len = bytes;
while (len >= 1024 && order < sizes.Length - 1)
{
order++;
len = len / 1024;
}
return $"{len:0.##} {sizes[order]}";
}
private async Task<string> GetIndexFilePath(string folderId)
{
var doc = await _client.GetIndexAsync("index_bindings").Result.GetDocumentAsync<Models.Index>(folderId);
return Path.Combine(_indexBasePath, doc.FolderId);
}
private static string GenerateApiKey(int length = 64)
{
if (length <= 0)
{
@ -77,7 +211,7 @@ public class MeiliSearchService:IDisposable
return apiKey.ToString();
}
public virtual async Task EnsureRepositoryIndexExists()
private async Task EnsureRepositoryIndexExists()
{
Task.Delay(5000).Wait();
var indexes = _client.GetAllIndexesAsync().Result;
@ -90,7 +224,7 @@ public class MeiliSearchService:IDisposable
_client.CreateIndexAsync("index_bindings").Wait();
}
public virtual string GetMeilisearchBinaryName()
private string GetMeilisearchBinaryName()
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
@ -114,7 +248,7 @@ public class MeiliSearchService:IDisposable
throw new PlatformNotSupportedException("Current platform and architecture combination is not supported");
}
public virtual async Task StartMeilisearch()
private async Task StartMeilisearch()
{
var binaryName = GetMeilisearchBinaryName();
var binaryPath = Path.Combine(AppContext.BaseDirectory, binaryName);
@ -141,10 +275,10 @@ public class MeiliSearchService:IDisposable
var host = RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
? "localhost"
: "127.0.0.1";
var args = "--http-addr "+host+":" + _meiliConfiguration.MeiliPort
+ " --env development --db-path "
+ Path.Combine(AppContext.BaseDirectory, "db")
+ " --master-key " + _apiKey;
var args = "--http-addr " + host + ":" + _meiliConfiguration.MeiliPort
+ " --env development --db-path "
+ Path.Combine(AppContext.BaseDirectory, "db");
//+ " --master-key " + _apiKey; note: bring back, masterkey not working when compressingi ndexs
var processStartInfo = new ProcessStartInfo
{
@ -180,30 +314,35 @@ public class MeiliSearchService:IDisposable
}
}
public void CheckIfNeedDocumentSync(object? sender, NotifyCollectionChangedEventArgs e)
private void CheckIfNeedDocumentSync(object? sender, NotifyCollectionChangedEventArgs e)
{
CheckIfNeedDocumentSync(THRESHOLD);
}
public void CheckIfNeedDocumentSync(int? threshold = null)
private void CheckIfNeedDocumentSync(int? threshold = null)
{
threshold = threshold ?? 0;
if(_documentCollection.Count>=threshold)
{
_logger.LogInformation("Threshold reached, syncing metadata to server.");
var grouped = _documentCollection.GroupBy(pair => pair.Key)
.ToDictionary(group => group.Key, group => group.Select(pair => pair.Value).ToList());
foreach (var repository in grouped)
{
var repositoryIndex = _client.GetIndexAsync(repository.Key).Result;
var documents = _documentCollection.ToList();
_documentCollection.Clear();
var result = RetryAsync(() => repositoryIndex.AddDocumentsAsync(repository.Value, "id")).Result;
}
SyncDocumentsToServer();
}
}
public async Task<T> RetryAsync<T>(Func<Task<T>> action, int maxRetries = 3, int delayMilliseconds = 1000)
private void SyncDocumentsToServer()
{
var grouped = _documentCollection.GroupBy(pair => pair.Key)
.ToDictionary(group => group.Key, group => group.Select(pair => pair.Value).ToList());
foreach (var repository in grouped)
{
var repositoryIndex = _client.GetIndexAsync(repository.Key).Result;
var documents = _documentCollection.ToList();
_documentCollection.Clear();
var result = RetryAsync(() => repositoryIndex.AddDocumentsAsync(repository.Value, "id")).Result;
}
}
private async Task<T> RetryAsync<T>(Func<Task<T>> action, int maxRetries = 3, int delayMilliseconds = 1000)
{
int retryCount = 0;
while (true)
@ -225,7 +364,7 @@ public class MeiliSearchService:IDisposable
}
}
}
public static string[] GetPropertiesInCamelCase<T>()
private static string[] GetPropertiesInCamelCase<T>()
{
var properties = typeof(T).GetProperties(BindingFlags.Public | BindingFlags.Instance);
return properties
@ -233,7 +372,7 @@ public class MeiliSearchService:IDisposable
.ToArray();
}
public static string ToCamelCase(string input)
private static string ToCamelCase(string input)
{
if (string.IsNullOrEmpty(input) || char.IsLower(input[0]))
{
@ -247,6 +386,50 @@ public class MeiliSearchService:IDisposable
#region Public
public virtual MeilisearchUsageStats GetProcessResourceUsage()
{
if (process == null || process.HasExited)
{
return new MeilisearchUsageStats();
}
try
{
process.Refresh();
// CPU Usage
TimeSpan cpuUsage = process.TotalProcessorTime;
double cpuPercentage = cpuUsage.TotalMilliseconds / (Environment.ProcessorCount * process.TotalProcessorTime.TotalMilliseconds) * 100;
// Memory Usage (in bytes)
long memoryUsage = process.WorkingSet64;
// Disk Usage (in bytes) - reads and writes
long diskRead = process.StartInfo.RedirectStandardOutput ? process.StandardOutput.BaseStream.Length : 0;
long diskWrite = process.StartInfo.RedirectStandardError ? process.StandardError.BaseStream.Length : 0;
return new MeilisearchUsageStats
{
CpuPercentage = Math.Round(cpuPercentage, 2),
MemoryUsageBytes = memoryUsage,
DiskReadBytes = diskRead,
DiskWriteBytes = diskWrite,
ProcessId = process.Id,
ThreadCount = process.Threads.Count
};
}
catch (InvalidOperationException)
{
// Process has exited
return new MeilisearchUsageStats();
}
catch (Exception ex)
{
_logger.LogError($"Error getting process resource usage: {ex.Message}");
return new MeilisearchUsageStats();
}
}
public virtual bool IsMeilisearchRunning()
{
var processName = RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
@ -256,8 +439,29 @@ public class MeiliSearchService:IDisposable
return processes.Any();
}
public virtual async Task SetIndexEnabled(string indexName, bool enabled)
{
_logger.LogTrace($"Updating index '{indexName}' status to {enabled}...");
if(enabled)
{
await DecompressIndex(indexName);
}
else
{
await CompressIndex(indexName);
}
_logger.LogInformation($"Updated index '{indexName}' status to {enabled}.");
}
public virtual void CreateIndex<T>(string indexName) where T : IDocument
{
if(GetAllIndexes().Count>=1000)
{
_logger.LogWarning("Maximum number of indexes reached, cannot create new index.");
return;
}
var indexes = _client.GetAllIndexesAsync().Result;
if (indexes.Results.Any(x => x.Uid == indexName))
{
@ -283,7 +487,10 @@ public class MeiliSearchService:IDisposable
{
Name = indexName,
CreatedAt = DateTime.UtcNow,
FolderId = folder
SizeBeforeCompression = null,
FolderId = folder,
IsCompressed = false,
LastCompressedAt = null
}
}, "name").Wait();
}
@ -303,11 +510,15 @@ public class MeiliSearchService:IDisposable
_logger.LogInformation($"Deleted index '{indexName}'!");
}
public void AddDocument(string repositoryId, IDocument document)
public void AddDocument(string repositoryId, IDocument document, bool autoCommit = false)
{
_logger.LogTrace($"Adding document '{document.Id}' to repository '{repositoryId}'...");
_documentCollection.Add(new KeyValuePair<string, IDocument>(repositoryId, document));
_logger.LogInformation($"Document {document.Id} added to collection.");
if (autoCommit)
{
SyncDocumentsToServer();
}
}
public virtual List<string> GetAllIndexes()
@ -318,6 +529,50 @@ public class MeiliSearchService:IDisposable
return result;
}
public virtual long GetIndexStorageUsage(string indexName, bool useCompressedSize = true)
{
var doc = _client.GetIndexAsync("index_bindings").Result.GetDocumentAsync<Models.Index>(indexName).Result;
if (doc.IsCompressed)
{
if(!useCompressedSize)
return doc.SizeBeforeCompression ?? 0;
var indexPath = GetIndexFilePath(indexName).Result+".zip";
if (!File.Exists(indexPath))
{
_logger.LogWarning($"Compressed index not found at: {indexPath}");
return 0;
}
return new FileInfo(indexPath).Length;
}
var path = Path.Combine(_indexBasePath, doc.FolderId);
if (!Directory.Exists(path))
{
_logger.LogWarning($"Index directory not found at: {path}");
return 0;
}
return new DirectoryInfo(path).GetFiles().Sum(f => f.Length);
}
public virtual long GetTotalStorageUsage(bool useCompressedSize = true)
{
var result = _client.GetIndexAsync("index_bindings").Result.GetDocumentsAsync<Models.Index>(new DocumentsQuery(){Limit = 1000}).Result;
var total = 0L;
foreach (var index in result.Results)
{
var indexPath = GetIndexFilePath(index.Name).Result+".zip";
if (index.IsCompressed)
if (useCompressedSize)
total += new FileInfo(indexPath).Length;
else
total += index.SizeBeforeCompression ?? 0;
else
total += index.IsCompressed ? index.SizeBeforeCompression ?? 0 : new DirectoryInfo(Path.Combine(_indexBasePath, index.FolderId)).GetFiles().Sum(f => f.Length);
}
return total;
}
public async void Start()
{
await StartMeilisearch();
@ -335,4 +590,5 @@ public class MeiliSearchService:IDisposable
_httpClient.Dispose();
}
#endregion
}
}

5
meilisearch.NET/Models/Index.cs
View File

@ -5,6 +5,9 @@ public class Index
public string Name { get; set; } = string.Empty;
public DateTime CreatedAt { get; set; } = DateTime.UtcNow;
public string FolderId { get; set; } = string.Empty;
public bool IsCompressed { get; set; } = false;
public DateTime? LastCompressedAt { get; set; }
public long? SizeBeforeCompression { get; set; } = 0;
public Index() { }
@ -14,4 +17,4 @@ public class Index
FolderId = folderId;
CreatedAt = createdAt;
}
}
}

11
meilisearch.NET/Models/MeilisearchUsageStats.cs Normal file
View File

@ -0,0 +1,11 @@
namespace meilisearch.NET.Models;
public class MeilisearchUsageStats
{
public double CpuPercentage { get; set; }
public long MemoryUsageBytes { get; set; }
public long DiskReadBytes { get; set; }
public long DiskWriteBytes { get; set; }
public int ThreadCount { get; set; }
public int ProcessId { get; set; }
}

BIN
meilisearch.NET/data.mdb Normal file
View File

Binary file not shown.

BIN
meilisearch.NET/lock.mdb Normal file
View File

Binary file not shown.

8
meilisearch.NET/meilisearch.NET.csproj
View File

@ -29,6 +29,14 @@
<Content Include="meilisearch-windows.exe">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</Content>
<None Remove="data.mdb" />
<Content Include="data.mdb">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</Content>
<None Remove="lock.mdb" />
<Content Include="lock.mdb">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</Content>
</ItemGroup>
<ItemGroup>