fix: improved logging + implemented methods for interacting with the meiliesearch

This commit is contained in:
Damien 2025-02-25 00:45:13 -05:00
parent 6db978d04c
commit fd4f5e594a
6 changed files with 327 additions and 338 deletions

View File

@ -1,36 +1,71 @@
using System.Net;
using System;
using System.Net;
using System.Threading.Tasks;
using meilisearch.NET;
using meilisearch.NET.Configurations;
using meilisearch.NET.Extensions;
using meilisearch.NET.Interfaces;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
ServicePointManager.SecurityProtocol = SecurityProtocolType.SystemDefault;
var builder = Host.CreateApplicationBuilder();
builder.Configuration.AddJsonFile("appsettings.json", optional: false, reloadOnChange: true);
public class Program
{
public static async Task Main(string[] args)
{
// Set security protocol to SystemDefault
ServicePointManager.SecurityProtocol = SecurityProtocolType.SystemDefault;
builder.Services.AddMeiliSearchService();
builder.Services.AddSingleton<test>();
// Configure and build the host
IHost host = CreateHostBuilder(args).Build();
builder.Logging.ClearProviders();
builder.Logging.AddConsole();
builder.Logging.SetMinimumLevel(LogLevel.Information);
//Resolve test dependency
var testService = host.Services.GetService<test>();
builder.Services.AddLogging();
var app = builder.Build();
// Run the host
await host.RunAsync();
}
app.Services.GetService<test>();
public static IHostBuilder CreateHostBuilder(string[] args) =>
Host.CreateDefaultBuilder(args)
.ConfigureAppConfiguration((hostingContext, configuration) =>
{
configuration.AddJsonFile("appsettings.json", optional: false, reloadOnChange: true);
configuration.AddEnvironmentVariables(); // Add environment variables as well (optional, but good practice)
configuration.AddCommandLine(args); // Support command line arguments
})
.ConfigureServices((hostContext, services) =>
{
services.AddMeiliSearchService();
services.AddSingleton<test>();
app.Run();
Console.ReadLine();
// Add logging configuration
services.AddLogging(builder =>
{
builder.ClearProviders();
builder.AddConsole();
builder.SetMinimumLevel(LogLevel.Information);
});
})
.UseConsoleLifetime(options =>
{
options.SuppressStatusMessages = true; // This is optional: you can suppress the "Application started" message
});
}
public class test
{
public test(IMeiliSearchService service)
private readonly ILogger<test> _logger;
public test(MeilisearchService service, ILogger<test> logger)
{
service.Start();
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
// You can perform actions with the Meilisearch service here
_logger.LogInformation("Test service initialized.");
}
}

View File

@ -1,16 +1,15 @@
using meilisearch.NET.Configurations;
using meilisearch.NET.Interfaces;
using Microsoft.Extensions.DependencyInjection;
using meilisearch.NET;
namespace meilisearch.NET.Extensions;
public static class ServiceCollectionExtension
{
public static IServiceCollection AddMeiliSearchService(this IServiceCollection services)
{
services.AddHttpClient<MeiliSearchService>();
services.AddHttpClient<MeilisearchService>();
services.AddSingleton<MeiliSearchConfiguration>();
services.AddSingleton<IMeiliSearchService, MeiliSearchService>();
services.AddSingleton<MeilisearchService>();
return services;
}
}

View File

@ -0,0 +1,6 @@
namespace meilisearch.NET.Interfaces;
public class IDocument
{
public Guid Id { get; set; }
}

View File

@ -1,74 +0,0 @@
using Meilisearch;
using meilisearch.NET.Enums;
namespace meilisearch.NET.Interfaces;
public interface IMeiliSearchService
{
/// <summary>
/// Checks if the Meilisearch server is healthy.
/// </summary>
/// <returns>True if healthy, false if not.</returns>
public bool HealthCheck(object? state);
/// <summary>
/// This is the SDK for Meilisearch. It is used to interact with the Meilisearch server.
/// </summary>
public MeilisearchClient Sdk { get; set; }
/// <summary>
/// This is the status of the Meilisearch server. It is used to check if the server is running or not.
/// </summary>
public MeiliSearchStatus Status { get; set; }
/// <summary>
/// Refreshes the API key. This is useful if you want to change the API key for security reasons.
/// </summary>
public void RefreshApiKey();
/// <summary>
/// This will start the Meilisearch server. This is useful if you want to start the server without starting the application.
/// </summary>
public void Start();
/// <summary>
/// This will restart the Meilisearch server. This is useful if you have made changes to the server and want to apply them without restarting the application.
/// </summary>
public void Restart();
/// <summary>
/// This will stop the Meilisearch server. This is useful if you want to stop the server without stopping the application.
/// </summary>
public void Stop();
/// <summary>
/// This will list all of the indexes regardless if they are loaded or unloaded.
/// </summary>
/// <returns></returns>
public List<string> ListIndexs();
/// <summary>
/// Creates a index and adds a .name file so that we can track the name of the index.
/// </summary>
/// <returns></returns>
public Meilisearch.Index CreateIndex();
/// <summary>
/// Loads a index and uncompresses it.
/// </summary>
/// <param name="indexId">The ID of the index to load.</param>
public void LoadIndex(string indexId);
/// <summary>
/// Unloads a index and compresses it.
/// </summary>
/// <param name="indexId">The ID of the index to unload.</param>
public void UnloadIndex(string indexId);
/// <summary>
/// This will delete a index and remove the .name file.
/// </summary>
/// <param name="indexId">The ID of the index to delete.</param>
public void DeleteIndex(string indexId);
}

View File

@ -1,3 +1,5 @@
using System.Collections.ObjectModel;
using System.Collections.Specialized;
using System.Diagnostics;
using System.Net.Sockets;
using System.Runtime.InteropServices;
@ -13,270 +15,285 @@ using Index = Meilisearch.Index;
namespace meilisearch.NET;
public class MeiliSearchService:IMeiliSearchService
public class MeilisearchService:IDisposable
{
public MeilisearchClient Sdk { get; set; }
public MeiliSearchStatus Status { get; set; }
private readonly MeiliSearchConfiguration _meiliSearchConfiguration;
private readonly ProcessStartInfo _processStartInfo;
private readonly ILogger<MeiliSearchService> _logger;
private readonly HttpClient _sdkHttpClient;
private Timer _healthCheckTimer;
private Process? _process;
public MeiliSearchService(HttpClient httpClient, ILogger<MeiliSearchService> logger, MeiliSearchConfiguration meiliSearchConfiguration)
private readonly HttpClient _httpClient;
private readonly ILogger<MeilisearchService> _logger;
public readonly MeilisearchClient Client;
private readonly MeiliSearchConfiguration _meiliConfiguration;
private Process process;
private ObservableCollection<KeyValuePair<string,IDocument>> _documentCollection;
private const int THRESHOLD = 10000;
private readonly List<string> FIELDS = new List<string>
{
"id",
"path",
"createdAtUtc",
"updatedAtUtc",
"lastAccessedAtUtc",
"name",
"type",
"ext",
"size"
};
public MeilisearchService(HttpClient httpClient, ILogger<MeilisearchService> logger, MeiliSearchConfiguration meiliConfiguration)
{
_httpClient = httpClient;
_meiliConfiguration = meiliConfiguration;
_logger = logger;
_meiliSearchConfiguration = meiliSearchConfiguration;
var binaryName = GetBinaryName();
MakeExecutable(Path.Combine(AppContext.BaseDirectory, binaryName));
string apiKey = "";
if (_meiliSearchConfiguration.EnableCustomApiKey)
apiKey = _meiliSearchConfiguration.ApiKey;
else
apiKey = ApiKeyGenerator.GenerateApiKey();
_sdkHttpClient = httpClient;
_sdkHttpClient.BaseAddress = new Uri("http://localhost:"+meiliSearchConfiguration.MeiliPort);
Sdk = new MeilisearchClient(httpClient, apiKey);
var path = Path.Combine(AppContext.BaseDirectory, binaryName);
var args = "--http-addr 127.0.0.1:" + _meiliSearchConfiguration.MeiliPort
+ " --master-key "+apiKey
+ " --env "+(_meiliSearchConfiguration.UiEnabled ? "development" : "production")
+" --db-path "+GetDataLocation();
_processStartInfo = new ProcessStartInfo
Client = new MeilisearchClient("http://localhost:"+meiliConfiguration.MeiliPort );
_documentCollection = new ObservableCollection<KeyValuePair<string,IDocument>>();
_documentCollection.CollectionChanged += CheckIfNeedDocumentSync;
Initialize();
}
#region Private
private async void Initialize()
{
await StartMeilisearch();
EnsureMeilisearchIsRunning();
EnsureRepositoryIndexExists();
}
private async void EnsureRepositoryIndexExists()
{
Task.Delay(5000).Wait();
var indexes = Client.GetAllIndexesAsync().Result;
if (indexes.Results.Any(x => x.Uid == "index_bindings"))
{
FileName = path,
_logger.LogInformation("index bindings already exists, skipping creation of index.");
return;
}
_logger.LogInformation("Creating index bindings for SDK to track indexs...");
Client.CreateIndexAsync("index_bindings").Wait();
}
private string GetMeilisearchBinaryName()
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
return "meilisearch-windows.exe";
}
if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
{
return RuntimeInformation.ProcessArchitecture == Architecture.Arm64
? "meilisearch-macos-arm"
: "meilisearch-macos-x64";
}
if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
{
return RuntimeInformation.ProcessArchitecture == Architecture.Arm64
? "meilisearch-linux-arm"
: "meilisearch-linux-x64";
}
throw new PlatformNotSupportedException("Current platform and architecture combination is not supported");
}
private void EnsureMeilisearchIsRunning()
{
if (!IsMeilisearchRunning())
{
StartMeilisearch().Wait();
}
}
private bool IsMeilisearchRunning()
{
var processName = RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
? "meilisearch-windows"
: "meilisearch";
var processes = Process.GetProcessesByName(processName);
return processes.Any();
}
private async Task StartMeilisearch()
{
var binaryName = GetMeilisearchBinaryName();
var binaryPath = Path.Combine(AppContext.BaseDirectory, binaryName);
if (!File.Exists(binaryPath))
{
_logger.LogError($"Meilisearch binary not found at: {binaryPath}");
throw new FileNotFoundException($"Could not find Meilisearch binary: {binaryName}");
}
// Set execute permissions on Unix-like systems
if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
try
{
var chmod = Process.Start("chmod", $"+x {binaryPath}");
chmod?.WaitForExit();
}
catch (Exception ex)
{
_logger.LogWarning($"Failed to set execute permissions on binary: {ex.Message}");
}
}
var host = RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
? "localhost"
: "127.0.0.1";
var args = "--http-addr "+host+":" + _meiliConfiguration.MeiliPort
+ " --env development --db-path "
+ Path.Combine(AppContext.BaseDirectory, "db");
var processStartInfo = new ProcessStartInfo
{
FileName = binaryPath,
Arguments = args,
UseShellExecute = false,
RedirectStandardOutput = true,
RedirectStandardError = true,
CreateNoWindow = true
};
Status = MeiliSearchStatus.Stopped;
_logger?.LogInformation($"Meilisearch service initialized with the UI {(_meiliSearchConfiguration.UiEnabled ? "Enabled" : "Disabled)")} with database located at {GetDataLocation()}.");
}
private static void MakeExecutable(string filePath)
{
if (string.IsNullOrWhiteSpace(filePath))
{
throw new ArgumentException("File path cannot be null or empty.", nameof(filePath));
}
// Check if the file exists
if (!File.Exists(filePath))
{
throw new FileNotFoundException($"The file '{filePath}' does not exist.", filePath);
}
if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux) || RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
{
// For Linux and macOS, use chmod +x
ExecuteCommand($"chmod +x \"{filePath}\"");
}
else if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
}
else
{
throw new PlatformNotSupportedException("This platform is not supported.");
}
Console.WriteLine($"File '{filePath}' has been made executable.");
}
private static void ExecuteCommand(string command)
{
var processStartInfo = new ProcessStartInfo
{
FileName = "/bin/bash", // Use bash for Linux and macOS
Arguments = $"-c \"{command}\"",
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false,
CreateNoWindow = true
};
using (var process = Process.Start(processStartInfo))
{
process.WaitForExit();
// Optionally handle output and errors
string output = process.StandardOutput.ReadToEnd();
string error = process.StandardError.ReadToEnd();
if (process.ExitCode != 0)
{
throw new InvalidOperationException($"Error executing command: {error}");
}
}
}
private static string GetDataLocation()
{
return Path.Combine(AppContext.BaseDirectory, "data");
string appDataDirectory;
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
appDataDirectory = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "MeiliSearchEmbedded");
}
else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
{
appDataDirectory = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.Personal), "Library", "Application Support", "MeiliSearchEmbedded");
}
else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
{
appDataDirectory = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.Personal), ".config", "MeiliSearchEmbedded");
}
else
{
throw new PlatformNotSupportedException("Unsupported OS.");
}
return appDataDirectory;
}
private static string GetBinaryName()
{
string os = RuntimeInformation.OSDescription.ToLowerInvariant();
string architecture = RuntimeInformation.ProcessArchitecture.ToString().ToLowerInvariant();
string platform;
if (os.Contains("linux"))
{
platform = "meilisearch-linux-" + (architecture.Contains("arm") ? "arm" : "x64");
}
else if (os.Contains("macos") || os.Contains("darwin"))
{
platform = "meilisearch-macos-" + (architecture.Contains("arm") ? "arm" : "x64");
}
else if (os.Contains("windows"))
{
platform = "meilisearch-windows.exe";
}
else
{
throw new PlatformNotSupportedException("Operating system not supported.");
}
return platform;
}
private bool MeilisSearchReachable()
{
string host = "127.0.0.1"; // localhost
int port = _meiliSearchConfiguration.MeiliPort;
try
{
using (TcpClient client = new TcpClient())
{
client.Connect(host, port);
return true;
}
}
catch
{
return false;
}
}
public bool HealthCheck(object? state)
{
if (_process == null)
throw new Exception("Meilisearch process is not running.");
var responding = _process.Responding;
var exited = _process.HasExited || _process.ExitCode == 0;
var reachable = MeilisSearchReachable();
return responding && exited && reachable;
}
public void RefreshApiKey()
{
_logger.LogInformation("Refreshing API key.");
string apiKey = "";
if (_meiliSearchConfiguration.EnableCustomApiKey)
apiKey = _meiliSearchConfiguration.ApiKey;
else
apiKey = ApiKeyGenerator.GenerateApiKey();
Sdk = new MeilisearchClient(_sdkHttpClient, apiKey);
_logger.LogInformation("New API key generated.");
Restart();
}
public void Start()
{
_logger.LogInformation("Starting MeiliSearch process.");
Status = MeiliSearchStatus.Starting;
_process = new Process { StartInfo = _processStartInfo };
process = new Process { StartInfo = processStartInfo };
try
{
_process.Start();
Status = MeiliSearchStatus.Running;
_logger.LogInformation("MeiliSearch process started successfully.");
process.Start();
await Task.Delay(5000); // Wait for the process to start
_logger.LogInformation($"Started Meilisearch process using binary: {binaryName}");
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to start MeiliSearch process.");
_logger.LogError($"Failed to start Meilisearch: {ex.Message}");
throw;
}
}
public void Restart()
private void CheckIfNeedDocumentSync(object? sender, NotifyCollectionChangedEventArgs e)
{
_logger.LogInformation("Restarting MeiliSearch process.");
CheckIfNeedDocumentSync(THRESHOLD);
}
private void CheckIfNeedDocumentSync(int? threshold = null)
{
threshold = threshold ?? 0;
if(_documentCollection.Count>=threshold)
{
_logger.LogInformation("Threshold reached, syncing metadata to server.");
var grouped = _documentCollection.GroupBy(pair => pair.Key)
.ToDictionary(group => group.Key, group => group.Select(pair => pair.Value).ToList());
foreach (var repository in grouped)
{
var repositoryIndex = Client.GetIndexAsync(repository.Key).Result;
var documents = _documentCollection.ToList();
_documentCollection.Clear();
var result = RetryAsync(() => repositoryIndex.AddDocumentsAsync(repository.Value, "id")).Result;
}
}
}
private async Task<T> RetryAsync<T>(Func<Task<T>> action, int maxRetries = 3, int delayMilliseconds = 1000)
{
int retryCount = 0;
while (true)
{
try
{
return await action();
}
catch (Exception ex)
{
retryCount++;
if (retryCount >= maxRetries)
{
_logger.LogError($"Operation failed after {maxRetries} retries: {ex.Message}");
throw;
}
_logger.LogWarning($"Operation failed, retrying {retryCount}/{maxRetries}...");
await Task.Delay(delayMilliseconds);
}
}
}
#endregion
#region Public
/// <summary>
/// Creates a new index on the Meilisearch server if it does not already exist.
/// </summary>
/// <param name="indexName">The name for the new index.</param>
public void CreateIndex(string indexName)
{
var indexes = Client.GetAllIndexesAsync().Result;
if (indexes.Results.Any(x => x.Uid == indexName))
{
_logger.LogWarning($"Index {indexName} already exists, skipping creation of index.");
return;
}
_logger.LogTrace($"Creating index '{indexName}'...");
Client.CreateIndexAsync(indexName).Wait();
Task.Delay(5000).Wait();
var index = Client.GetIndexAsync(indexName).Result;
var test = index.GetFilterableAttributesAsync().Result;
index.UpdateFilterableAttributesAsync(FIELDS).Wait();
_logger.LogInformation($"{indexName} index created!");
Client.GetIndexAsync("index_bindings").Result.AddDocumentsAsync(new List<Models.Index>
{
new Models.Index()
{
Name = indexName
}
}, "name").Wait();
}
/// <summary>
/// Deletes a index for a repoistory.
/// </summary>
/// <param name="indexName">The name of the index.</param>
public void DeleteIndex(string indexName)
{
var indexes = Client.GetAllIndexesAsync().Result;
if (indexes.Results.Any(x => x.Uid != indexName))
{
_logger.LogWarning($"Index '{indexName}' does not exist, skipping deletion of index.");
return;
}
_logger.LogTrace($"Deleting index '{indexName}'...");
Client.DeleteIndexAsync(indexName).Wait();
Client.GetIndexAsync("index_bindings").Result.DeleteOneDocumentAsync(indexName).Wait();
_logger.LogInformation($"Deleted index '{indexName}'!");
}
public void AddDocument(string repositoryId, IDocument document)
{
_logger.LogTrace($"Adding document '{document.Id}' to repository '{repositoryId}'...");
_documentCollection.Add(new KeyValuePair<string, IDocument>(repositoryId, document));
_logger.LogInformation($"Document {document.Id} added to collection.");
}
public List<string> GetAllIndexes()
{
_logger.LogTrace("Fetching all indexes from Meilisearch server created with the SDK...");
var result = Client.GetAllIndexesAsync().Result.Results.Select(x => x.Uid).Where(x=>x!="index_bindings").ToList();
_logger.LogInformation($"Fetched {result.Count} indexes from Meilisearch server.");
return result;
}
public async void Start()
{
await StartMeilisearch();
}
public async void Stop()
{
process.Kill();
}
public void Dispose()
{
CheckIfNeedDocumentSync();
Stop();
Start();
}
public void Stop()
{
if (_process == null)
{
_logger.LogWarning("Attempted to stop MeiliSearch process, but it is not running.");
throw new Exception("Meilisearch process is not running.");
}
_logger.LogInformation("Stopping MeiliSearch process.");
Status = MeiliSearchStatus.Stopping;
try
{
_process.Kill();
Status = MeiliSearchStatus.Stopped;
_logger.LogInformation("MeiliSearch process stopped successfully.");
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to stop MeiliSearch process.");
throw;
}
}
public List<string> ListIndexs()
{
throw new NotImplementedException();
}
public Index CreateIndex()
{
throw new NotImplementedException();
}
public void LoadIndex(string indexId)
{
throw new NotImplementedException();
}
public void UnloadIndex(string indexId)
{
throw new NotImplementedException();
}
public void DeleteIndex(string indexId)
{
throw new NotImplementedException();
_httpClient.Dispose();
}
#endregion
}

View File

@ -0,0 +1,6 @@
namespace meilisearch.NET.Models;
public class Index
{
public string Name { get; set; }
}