feat: meilisearch index enable/disable automatic compression + usage method

meilisearch.NET.example/Program.cs

@@ -62,42 +62,52 @@ public class test
             Task.Delay(1000).Wait(); // Wait for 1 second before checking again
         }
 
-        service.CreateIndex<document>("test");
+        var usage = service.GetProcessResourceUsage();
-        service.AddDocument("test", new document()
+        _logger.LogInformation($"Memory usage: {usage.MemoryUsageBytes} MB");
-        {
+        _logger.LogInformation($"CPU usage: {usage.CpuPercentage} %");
-            Id = Guid.NewGuid(),
+        _logger.LogInformation($"Disk read: {usage.DiskReadBytes} MB");
-            message = "Hello, Meilisearch!"
+        _logger.LogInformation($"Disk write: {usage.DiskWriteBytes} MB");
-        });
+        _logger.LogInformation($"Thread count: {usage.ThreadCount}");
-        service.AddDocument("test", new document()
+        _logger.LogInformation($"Process ID: {usage.ProcessId}");
-        {
+        
-            Id = Guid.NewGuid(),
+        //service.UpdateIndexStatus("test",false).Wait();
-            message = "Hello, Meilisearch!"
+        
-        });
+        
-        service.AddDocument("test", new document()
+        // service.AddDocument("test", new document()
-        {
+        // {
-            Id = Guid.NewGuid(),
+        //     Id = Guid.NewGuid(),
-            message = "Hello, Meilisearch!"
+        //     message = "Hello, Meilisearch!"
-        });
+        // });
-        service.AddDocument("test", new document()
+        // service.AddDocument("test", new document()
-        {
+        // {
-            Id = Guid.NewGuid(),
+        //     Id = Guid.NewGuid(),
-            message = "Hello, Meilisearch!"
+        //     message = "Hello, Meilisearch!"
-        });
+        // });
-        service.AddDocument("test", new document()
+        // service.AddDocument("test", new document()
-        {
+        // {
-            Id = Guid.NewGuid(),
+        //     Id = Guid.NewGuid(),
-            message = "Hello, Meilisearch!"
+        //     message = "Hello, Meilisearch!"
-        });
+        // });
-        service.AddDocument("test", new document()
+        // service.AddDocument("test", new document()
-        {
+        // {
-            Id = Guid.NewGuid(),
+        //     Id = Guid.NewGuid(),
-            message = "Hello, Meilisearch!"
+        //     message = "Hello, Meilisearch!"
-        });
+        // });
-        service.AddDocument("test", new document()
+        // service.AddDocument("test", new document()
-        {
+        // {
-            Id = Guid.NewGuid(),
+        //     Id = Guid.NewGuid(),
-            message = "Hello, Meilisearch!"
+        //     message = "Hello, Meilisearch!"
-        });
+        // });
+        // service.AddDocument("test", new document()
+        // {
+        //     Id = Guid.NewGuid(),
+        //     message = "Hello, Meilisearch!"
+        // });
+        // service.AddDocument("test", new document()
+        // {
+        //     Id = Guid.NewGuid(),
+        //     message = "Hello, Meilisearch!"
+        // });
         _logger.LogInformation("Test service initialized.");
     }
 }

meilisearch.NET/MeiliSearchService.cs

@@ -1,6 +1,7 @@
 using System.Collections.ObjectModel;
 using System.Collections.Specialized;
 using System.Diagnostics;
+using System.IO.Compression;
 using System.Net.Sockets;
 using System.Reflection;
 using System.Runtime.InteropServices;
@@ -12,6 +13,8 @@ using meilisearch.NET.Configurations;
 using meilisearch.NET.Enums;
 using meilisearch.NET.Extensions;
 using meilisearch.NET.Interfaces;
+using meilisearch.NET.Models;
+using Meilisearch.QueryParameters;
 using Microsoft.Extensions.Logging;
 using Index = Meilisearch.Index;
 
@@ -20,15 +23,19 @@ namespace meilisearch.NET;
 
 public class MeiliSearchService:IDisposable
 {
-    public readonly HttpClient _httpClient;
+    private readonly HttpClient _httpClient;
-    public readonly ILogger<MeiliSearchService> _logger;
+    private readonly ILogger<MeiliSearchService> _logger;
-    public readonly MeilisearchClient _client;
+    private readonly MeilisearchClient _client;
-    public readonly MeiliSearchConfiguration _meiliConfiguration;
+    private readonly MeiliSearchConfiguration _meiliConfiguration;
-    public readonly string _indexBasePath = Path.Combine(AppContext.BaseDirectory, "db", "indexes" );
+    private readonly string _indexBasePath = Path.Combine(AppContext.BaseDirectory, "db", "indexes" );
-    public static string _apiKey = GenerateApiKey();
+    private static string _apiKey = GenerateApiKey();
-    public const int THRESHOLD = 10000;
+    private const int THRESHOLD = 10000;
-    public Process? process;
+    private const string DEFAULT_DATA_FILE_PATH = "data.mdb";
-    public ObservableCollection<KeyValuePair<string,IDocument>> _documentCollection;
+    private const string DEFAULT_LOCK_FILE_PATH = "lock.mdb";
+    private Process? process;
+    private ObservableCollection<KeyValuePair<string,IDocument>> _documentCollection;
+    private List<Index> indexes { get; set; } = new();
+    
     protected virtual ObservableCollection<KeyValuePair<string, IDocument>> DocumentCollection
     {
         get
@@ -54,7 +61,134 @@ public class MeiliSearchService:IDisposable
 
     
     #region Private       
-    public static string GenerateApiKey(int length = 64)
+    private async Task CompressIndex(string indexName)
+    {
+        var indexPath = await GetIndexFilePath(indexName);
+        if (!Directory.Exists(indexPath))
+        {
+            _logger.LogWarning($"Index directory not found at: {indexPath}");
+            return;
+        }
+
+        var compressedPath = indexPath + ".zip";
+        _logger.LogTrace($"Compressing index '{indexName}' to {compressedPath}...");
+        
+        try
+        {
+            var size = new DirectoryInfo(indexPath).GetFiles().Sum(f => f.Length);
+            // Create temp directory to ensure we don't lose data if compression fails
+            var tempPath = compressedPath + ".temp";
+            ZipFile.CreateFromDirectory(indexPath, tempPath, CompressionLevel.SmallestSize, false);
+            
+            // If compression succeeded, safely replace old zip (if exists) and remove original directory
+            if (File.Exists(compressedPath))
+            {
+                File.Delete(compressedPath);
+            }
+            File.Move(tempPath, compressedPath);
+            Directory.Delete(indexPath, true);
+
+            // Update index metadata
+            var indexBindings = await _client.GetIndexAsync("index_bindings");
+            var doc = await _client.GetIndexAsync("index_bindings").Result.GetDocumentAsync<Models.Index>(indexName);
+            var document = new Models.Index
+            {
+                Name = indexName,
+                IsCompressed = true,
+                FolderId = doc.FolderId,
+                CreatedAt = doc.CreatedAt,
+                SizeBeforeCompression = size,
+                LastCompressedAt = DateTime.UtcNow
+            };
+            await indexBindings.UpdateDocumentsAsync(new List<Models.Index> { document });
+            _logger.LogInformation($"Successfully compressed index '{indexName}'");
+            
+            Directory.CreateDirectory(indexPath);
+            File.Copy(Path.Combine(AppContext.BaseDirectory,DEFAULT_DATA_FILE_PATH), Path.Combine(indexPath, DEFAULT_DATA_FILE_PATH));
+            File.Copy(Path.Combine(AppContext.BaseDirectory,DEFAULT_LOCK_FILE_PATH), Path.Combine(indexPath, DEFAULT_LOCK_FILE_PATH));
+            _logger.LogInformation($"Created placeholder data file for compressed index '{indexName}'");
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError($"Failed to compress index '{indexName}': {ex.Message}");
+            throw;
+        }
+        Stop();
+    }
+
+    private async Task DecompressIndex(string indexName)
+    {
+        var compressedPath = await GetIndexFilePath(indexName) + ".zip";
+        var extractPath = await GetIndexFilePath(indexName);
+        
+
+        if (!File.Exists(compressedPath))
+        {
+            _logger.LogWarning($"Compressed index not found at: {compressedPath}");
+            return;
+        }
+
+        _logger.LogTrace($"Decompressing index '{indexName}' to {extractPath}...");
+        
+        try
+        {
+            if (Directory.Exists(extractPath))
+            {
+                Directory.Delete(extractPath, true);
+            }
+            
+            // Create temp directory to ensure we don't lose data if decompression fails
+            var tempPath = extractPath + ".temp";
+            ZipFile.ExtractToDirectory(compressedPath, tempPath);
+            
+            // If decompression succeeded, safely move to final location
+            if (Directory.Exists(extractPath))
+            {
+                Directory.Delete(extractPath, true);
+            }
+            Directory.Move(tempPath, extractPath);
+            File.Delete(compressedPath);
+
+            // Update index metadata
+            var indexBindings = await _client.GetIndexAsync("index_bindings");
+            var doc = await _client.GetIndexAsync("index_bindings").Result.GetDocumentAsync<Models.Index>(indexName);
+            var document = new Models.Index
+            {
+                Name = indexName,
+                FolderId = doc.FolderId,
+                CreatedAt = doc.CreatedAt,
+                SizeBeforeCompression = null,
+                IsCompressed = false,
+                LastCompressedAt = doc.LastCompressedAt
+            };
+            await indexBindings.UpdateDocumentsAsync(new List<Models.Index> { document });
+
+            _logger.LogInformation($"Successfully decompressed index '{indexName}'");
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError($"Failed to decompress index '{indexName}': {ex.Message}");
+            throw;
+        }
+    }  
+    private static string FormatBytes(long bytes)
+    {
+        string[] sizes = { "B", "KB", "MB", "GB", "TB" };
+        int order = 0;
+        double len = bytes;
+        while (len >= 1024 && order < sizes.Length - 1)
+        {
+            order++;
+            len = len / 1024;
+        }
+        return $"{len:0.##} {sizes[order]}";
+    }
+    private async Task<string> GetIndexFilePath(string folderId)
+    {
+        var doc = await _client.GetIndexAsync("index_bindings").Result.GetDocumentAsync<Models.Index>(folderId);
+        return Path.Combine(_indexBasePath, doc.FolderId);
+    }
+    private static string GenerateApiKey(int length = 64)
     {
         if (length <= 0)
         {
@@ -77,7 +211,7 @@ public class MeiliSearchService:IDisposable
 
         return apiKey.ToString();
     }
-    public virtual async Task EnsureRepositoryIndexExists()
+    private async Task EnsureRepositoryIndexExists()
     {
         Task.Delay(5000).Wait();
         var indexes = _client.GetAllIndexesAsync().Result;
@@ -90,7 +224,7 @@ public class MeiliSearchService:IDisposable
         _client.CreateIndexAsync("index_bindings").Wait();
     }
     
-    public virtual string GetMeilisearchBinaryName()
+    private string GetMeilisearchBinaryName()
     {
         if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
         {
@@ -114,7 +248,7 @@ public class MeiliSearchService:IDisposable
         throw new PlatformNotSupportedException("Current platform and architecture combination is not supported");
     }
 
-    public virtual async Task StartMeilisearch()
+    private async Task StartMeilisearch()
     {
         var binaryName = GetMeilisearchBinaryName();
         var binaryPath = Path.Combine(AppContext.BaseDirectory, binaryName);
@@ -143,8 +277,8 @@ public class MeiliSearchService:IDisposable
             : "127.0.0.1";
         var args = "--http-addr " + host + ":" + _meiliConfiguration.MeiliPort
                    + " --env development --db-path "
-                  + Path.Combine(AppContext.BaseDirectory, "db")
+                   + Path.Combine(AppContext.BaseDirectory, "db");
-                  + " --master-key " + _apiKey;
+                  //+ " --master-key " + _apiKey; note: bring back, masterkey not working when compressingi ndexs
 
         var processStartInfo = new ProcessStartInfo
         {
@@ -180,17 +314,23 @@ public class MeiliSearchService:IDisposable
         }
     }
     
-    public void CheckIfNeedDocumentSync(object? sender, NotifyCollectionChangedEventArgs e)
+    private void CheckIfNeedDocumentSync(object? sender, NotifyCollectionChangedEventArgs e)
     {
         CheckIfNeedDocumentSync(THRESHOLD);
     }
 
-    public void CheckIfNeedDocumentSync(int? threshold = null)
+    private void CheckIfNeedDocumentSync(int? threshold = null)
     {
         threshold = threshold ?? 0;
         if(_documentCollection.Count>=threshold)
         {
             _logger.LogInformation("Threshold reached, syncing metadata to server.");
+            SyncDocumentsToServer();
+        }
+    }
+
+    private void SyncDocumentsToServer()
+    {
         var grouped = _documentCollection.GroupBy(pair => pair.Key)
             .ToDictionary(group => group.Key, group => group.Select(pair => pair.Value).ToList());
         foreach (var repository in grouped)
@@ -201,9 +341,8 @@ public class MeiliSearchService:IDisposable
             var result = RetryAsync(() => repositoryIndex.AddDocumentsAsync(repository.Value, "id")).Result;
         }
     }
-    }
 
-    public async Task<T> RetryAsync<T>(Func<Task<T>> action, int maxRetries = 3, int delayMilliseconds = 1000)
+    private async Task<T> RetryAsync<T>(Func<Task<T>> action, int maxRetries = 3, int delayMilliseconds = 1000)
     {
         int retryCount = 0;
         while (true)
@@ -225,7 +364,7 @@ public class MeiliSearchService:IDisposable
             }
         }
     }
-    public static string[] GetPropertiesInCamelCase<T>()
+    private static string[] GetPropertiesInCamelCase<T>()
     {
         var properties = typeof(T).GetProperties(BindingFlags.Public | BindingFlags.Instance);
         return properties
@@ -233,7 +372,7 @@ public class MeiliSearchService:IDisposable
             .ToArray();
     }
 
-    public static string ToCamelCase(string input)
+    private static string ToCamelCase(string input)
     {
         if (string.IsNullOrEmpty(input) || char.IsLower(input[0]))
         {
@@ -247,6 +386,50 @@ public class MeiliSearchService:IDisposable
     
     
     #region Public
+    
+    public virtual MeilisearchUsageStats GetProcessResourceUsage()
+    {
+        if (process == null || process.HasExited)
+        {
+            return new MeilisearchUsageStats();
+        }
+
+        try
+        {
+            process.Refresh();
+            
+            // CPU Usage
+            TimeSpan cpuUsage = process.TotalProcessorTime;
+            double cpuPercentage = cpuUsage.TotalMilliseconds /    (Environment.ProcessorCount * process.TotalProcessorTime.TotalMilliseconds) * 100;
+
+            // Memory Usage (in bytes)
+            long memoryUsage = process.WorkingSet64;
+
+            // Disk Usage (in bytes) - reads and writes
+            long diskRead = process.StartInfo.RedirectStandardOutput ? process.StandardOutput.BaseStream.Length : 0;
+            long diskWrite = process.StartInfo.RedirectStandardError ? process.StandardError.BaseStream.Length : 0;
+
+            return new MeilisearchUsageStats
+            {
+                CpuPercentage = Math.Round(cpuPercentage, 2),
+                MemoryUsageBytes = memoryUsage,
+                DiskReadBytes = diskRead,
+                DiskWriteBytes = diskWrite,
+                ProcessId = process.Id,
+                ThreadCount = process.Threads.Count
+            };
+        }
+        catch (InvalidOperationException)
+        {
+            // Process has exited
+            return new MeilisearchUsageStats();
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError($"Error getting process resource usage: {ex.Message}");
+            return new MeilisearchUsageStats();
+        }
+    }
     public virtual bool IsMeilisearchRunning()
     {
         var processName = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) 
@@ -256,8 +439,29 @@ public class MeiliSearchService:IDisposable
         return processes.Any();
     }
 
+    public virtual async Task SetIndexEnabled(string indexName, bool enabled)
+    {
+        _logger.LogTrace($"Updating index '{indexName}' status to {enabled}...");
+        if(enabled)
+        {
+            await DecompressIndex(indexName);
+        }
+        else
+        {
+            await CompressIndex(indexName);
+        }
+        _logger.LogInformation($"Updated index '{indexName}' status to {enabled}.");
+    }
+    
+    
     public virtual void CreateIndex<T>(string indexName) where T : IDocument
     {
+        if(GetAllIndexes().Count>=1000)
+        {
+            _logger.LogWarning("Maximum number of indexes reached, cannot create new index.");
+            return;
+        }
+        
         var indexes = _client.GetAllIndexesAsync().Result;
         if (indexes.Results.Any(x => x.Uid == indexName))
         {
@@ -283,7 +487,10 @@ public class MeiliSearchService:IDisposable
                 {
                     Name = indexName,
                     CreatedAt = DateTime.UtcNow,
-                    FolderId = folder
+                    SizeBeforeCompression = null,
+                    FolderId = folder,
+                    IsCompressed = false,
+                    LastCompressedAt = null
                 }
             }, "name").Wait();
         }
@@ -303,11 +510,15 @@ public class MeiliSearchService:IDisposable
         _logger.LogInformation($"Deleted index '{indexName}'!");
     }
     
-    public void AddDocument(string repositoryId, IDocument document)
+    public void AddDocument(string repositoryId, IDocument document, bool autoCommit = false)
     {
         _logger.LogTrace($"Adding document '{document.Id}' to repository '{repositoryId}'...");
         _documentCollection.Add(new KeyValuePair<string, IDocument>(repositoryId, document));
         _logger.LogInformation($"Document {document.Id} added to collection.");
+        if (autoCommit)
+        {
+            SyncDocumentsToServer();
+        }
     }
 
     public virtual List<string> GetAllIndexes()
@@ -318,6 +529,50 @@ public class MeiliSearchService:IDisposable
         return  result;
     }
 
+    public virtual long GetIndexStorageUsage(string indexName, bool useCompressedSize = true)
+    {
+        var doc = _client.GetIndexAsync("index_bindings").Result.GetDocumentAsync<Models.Index>(indexName).Result;
+        
+        if (doc.IsCompressed)
+        {
+            if(!useCompressedSize)
+                return doc.SizeBeforeCompression ?? 0;    
+            
+            var indexPath = GetIndexFilePath(indexName).Result+".zip";
+            if (!File.Exists(indexPath))
+            {
+                _logger.LogWarning($"Compressed index not found at: {indexPath}");
+                return 0;
+            }
+            return new FileInfo(indexPath).Length;
+        }
+        var path = Path.Combine(_indexBasePath, doc.FolderId);
+        if (!Directory.Exists(path))
+        {
+            _logger.LogWarning($"Index directory not found at: {path}");
+            return 0;
+        }
+        return new DirectoryInfo(path).GetFiles().Sum(f => f.Length);
+    }
+    
+    public virtual long GetTotalStorageUsage(bool useCompressedSize = true)
+    {
+        var result = _client.GetIndexAsync("index_bindings").Result.GetDocumentsAsync<Models.Index>(new DocumentsQuery(){Limit = 1000}).Result;
+        var total = 0L;
+        foreach (var index in result.Results)
+        {
+            var indexPath = GetIndexFilePath(index.Name).Result+".zip";
+            if (index.IsCompressed)
+                if (useCompressedSize)
+                    total += new FileInfo(indexPath).Length;
+                else
+                    total += index.SizeBeforeCompression ?? 0;
+            else
+                total += index.IsCompressed ? index.SizeBeforeCompression ?? 0 : new DirectoryInfo(Path.Combine(_indexBasePath, index.FolderId)).GetFiles().Sum(f => f.Length);
+        } 
+        return total;
+    }
+    
     public async void Start()
     {
         await StartMeilisearch();
@@ -335,4 +590,5 @@ public class MeiliSearchService:IDisposable
         _httpClient.Dispose();
     }
     #endregion
+
 }

meilisearch.NET/Models/Index.cs

@@ -5,6 +5,9 @@ public class Index
     public string Name { get; set; } = string.Empty;
     public DateTime CreatedAt { get; set; } = DateTime.UtcNow;
     public string FolderId { get; set; } = string.Empty;
+    public bool IsCompressed { get; set; } = false;
+    public DateTime? LastCompressedAt { get; set; }
+    public long? SizeBeforeCompression { get; set; } = 0;
 
     public Index() { }
 

meilisearch.NET/Models/MeilisearchUsageStats.cs

@@ -0,0 +1,11 @@
+namespace meilisearch.NET.Models;
+
+public class MeilisearchUsageStats
+{
+    public double CpuPercentage { get; set; }
+    public long MemoryUsageBytes { get; set; }
+    public long DiskReadBytes { get; set; }
+    public long DiskWriteBytes { get; set; }
+    public int ThreadCount { get; set; }
+    public int ProcessId { get; set; }
+}

meilisearch.NET/meilisearch.NET.csproj

@@ -29,6 +29,14 @@
       <Content Include="meilisearch-windows.exe">
         <CopyToOutputDirectory>Always</CopyToOutputDirectory>
       </Content>
+      <None Remove="data.mdb" />
+      <Content Include="data.mdb">
+        <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+      </Content>
+      <None Remove="lock.mdb" />
+      <Content Include="lock.mdb">
+        <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+      </Content>
     </ItemGroup>
 
     <ItemGroup>