From 66f108ef53d83494421edce12bd5f9305a22e380 Mon Sep 17 00:00:00 2001 From: PaulKoudelka Date: Thu, 28 May 2026 12:36:21 +0200 Subject: [PATCH] added distinction between database and vectorstore --- .../Assistants/I18N/allTexts.lua | 31 ++-- .../Components/ChatComponent.razor.cs | 2 +- .../MindWork AI Studio.csproj | 6 +- .../Pages/Information.razor | 12 +- .../Pages/Information.razor.cs | 71 +++++---- .../DatabaseClientProvider.Qdrant.cs | 80 ++++++++++ .../Tools/Databases/DatabaseClientProvider.cs | 90 ++--------- .../Qdrant/QdrantClientImplementation.cs | 73 --------- .../Tools/Databases/VectorStoragePoint.cs | 16 ++ .../VectorStore/IVectorStoreClient.cs | 20 +++ .../VectorStore/NoVectorStoreClient.cs | 39 +++++ .../VectorStore/QdrantClientImplementation.cs | 142 ++++++++++++++++++ .../Metadata/MetaDataDatabasesAttribute.cs | 6 - .../Metadata/MetaDataVectorStoreAttribute.cs | 6 + 14 files changed, 382 insertions(+), 212 deletions(-) create mode 100644 app/MindWork AI Studio/Tools/Databases/DatabaseClientProvider.Qdrant.cs delete mode 100644 app/MindWork AI Studio/Tools/Databases/Qdrant/QdrantClientImplementation.cs create mode 100644 app/MindWork AI Studio/Tools/Databases/VectorStoragePoint.cs create mode 100644 app/MindWork AI Studio/Tools/Databases/VectorStore/IVectorStoreClient.cs create mode 100644 app/MindWork AI Studio/Tools/Databases/VectorStore/NoVectorStoreClient.cs create mode 100644 app/MindWork AI Studio/Tools/Databases/VectorStore/QdrantClientImplementation.cs delete mode 100644 app/MindWork AI Studio/Tools/Metadata/MetaDataDatabasesAttribute.cs create mode 100644 app/MindWork AI Studio/Tools/Metadata/MetaDataVectorStoreAttribute.cs diff --git a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua index 7020549e2..ea78f7328 100644 --- a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua +++ b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua @@ -6040,6 +6040,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1019424746"] = "Startup log file -- Browse AI Studio's source code on GitHub — we welcome your contributions. UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1107156991"] = "Browse AI Studio's source code on GitHub — we welcome your contributions." +-- Vector store version +UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1124039623"] = "Vector store version" + -- ID mismatch: the plugin ID differs from the enterprise configuration ID. UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1137744461"] = "ID mismatch: the plugin ID differs from the enterprise configuration ID." @@ -6052,9 +6055,6 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1290340974"] = "Unknown configur -- This library is used to read PDF files. This is necessary, e.g., for using PDFs as a data source for a chat. UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1388816916"] = "This library is used to read PDF files. This is necessary, e.g., for using PDFs as a data source for a chat." --- Database version -UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1420062548"] = "Database version" - -- This library is used to extend the MudBlazor library. It provides additional components that are not part of the MudBlazor library. UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1421513382"] = "This library is used to extend the MudBlazor library. It provides additional components that are not part of the MudBlazor library." @@ -6202,6 +6202,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T2924964415"] = "AI Studio runs w -- Changelog UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T3017574265"] = "Changelog" +-- Vector store +UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T3046399223"] = "Vector store" + -- Enterprise configuration ID: UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T3092349641"] = "Enterprise configuration ID:" @@ -6277,9 +6280,6 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T3986423270"] = "Check Pandoc Ins -- Versions UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T4010195468"] = "Versions" --- Database -UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T4036243672"] = "Database" - -- This library is used by the Rust runtime to read the current user's username, e.g. when an organization-managed ERI server uses the OS username for authentication. UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T4060906280"] = "This library is used by the Rust runtime to read the current user's username, e.g. when an organization-managed ERI server uses the OS username for authentication." @@ -7003,20 +7003,29 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::NODATABASECLIENT::T3662391977"] = " -- Status UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::NODATABASECLIENT::T6222351"] = "Status" +-- Reason +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::VECTORSTORE::NOVECTORSTORECLIENT::T1093747001"] = "Reason" + +-- Unavailable +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::VECTORSTORE::NOVECTORSTORECLIENT::T3662391977"] = "Unavailable" + +-- Status +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::VECTORSTORE::NOVECTORSTORECLIENT::T6222351"] = "Status" + -- Storage size -UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::QDRANT::QDRANTCLIENTIMPLEMENTATION::T1230141403"] = "Storage size" +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::VECTORSTORE::QDRANTCLIENTIMPLEMENTATION::T1230141403"] = "Storage size" -- HTTP port -UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::QDRANT::QDRANTCLIENTIMPLEMENTATION::T1717573768"] = "HTTP port" +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::VECTORSTORE::QDRANTCLIENTIMPLEMENTATION::T1717573768"] = "HTTP port" -- Reported version -UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::QDRANT::QDRANTCLIENTIMPLEMENTATION::T3556099842"] = "Reported version" +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::VECTORSTORE::QDRANTCLIENTIMPLEMENTATION::T3556099842"] = "Reported version" -- gRPC port -UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::QDRANT::QDRANTCLIENTIMPLEMENTATION::T757840040"] = "gRPC port" +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::VECTORSTORE::QDRANTCLIENTIMPLEMENTATION::T757840040"] = "gRPC port" -- Number of collections -UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::QDRANT::QDRANTCLIENTIMPLEMENTATION::T842647336"] = "Number of collections" +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::VECTORSTORE::QDRANTCLIENTIMPLEMENTATION::T842647336"] = "Number of collections" -- The related data is not allowed to be sent to any LLM provider. This means that this data source cannot be used at the moment. UI_TEXT_CONTENT["AISTUDIO::TOOLS::ERICLIENT::DATAMODEL::PROVIDERTYPEEXTENSIONS::T1555790630"] = "The related data is not allowed to be sent to any LLM provider. This means that this data source cannot be used at the moment." diff --git a/app/MindWork AI Studio/Components/ChatComponent.razor.cs b/app/MindWork AI Studio/Components/ChatComponent.razor.cs index ded3427f0..c8a8464ea 100644 --- a/app/MindWork AI Studio/Components/ChatComponent.razor.cs +++ b/app/MindWork AI Studio/Components/ChatComponent.razor.cs @@ -4,7 +4,7 @@ using AIStudio.Settings; using AIStudio.Settings.DataModel; using AIStudio.Tools.AIJobs; - +using AIStudio.Tools.Services; using Microsoft.AspNetCore.Components; using Microsoft.AspNetCore.Components.Web; diff --git a/app/MindWork AI Studio/MindWork AI Studio.csproj b/app/MindWork AI Studio/MindWork AI Studio.csproj index 7cebafb93..5421ffc9e 100644 --- a/app/MindWork AI Studio/MindWork AI Studio.csproj +++ b/app/MindWork AI Studio/MindWork AI Studio.csproj @@ -88,7 +88,7 @@ $([System.String]::Copy( $(Metadata) ).Split( ';' )[ 8 ]) $([System.String]::Copy( $(Metadata) ).Split( ';' )[ 9 ]) $([System.String]::Copy( $(Metadata) ).Split( ';' )[ 10 ]) - $([System.String]::Copy( $(Metadata) ).Split( ';' )[ 11 ]) + $([System.String]::Copy( $(Metadata) ).Split( ';' )[ 11 ]) true @@ -116,8 +116,8 @@ <_Parameter1>$(MetaPdfiumVersion) - - <_Parameter1>$(MetaQdrantVersion) + + <_Parameter1>$(MetaVectorStoreVersion) diff --git a/app/MindWork AI Studio/Pages/Information.razor b/app/MindWork AI Studio/Pages/Information.razor index 13f2e941a..29c4e4736 100644 --- a/app/MindWork AI Studio/Pages/Information.razor +++ b/app/MindWork AI Studio/Pages/Information.razor @@ -21,11 +21,11 @@ - @this.VersionDatabase + @this.VersionVectorStore - + - @foreach (var item in this.databaseDisplayInfo) + @foreach (var item in this.vectorStoreDisplayInfo) {
@@ -35,11 +35,11 @@ } - - @(this.showDatabaseDetails ? T("Hide Details") : T("Show Details")) + OnClick="@this.ToggleVectorStoreDetails"> + @(this.showVectorStoreDetails ? T("Hide Details") : T("Show Details")) diff --git a/app/MindWork AI Studio/Pages/Information.razor.cs b/app/MindWork AI Studio/Pages/Information.razor.cs index 9f7250ac3..7c018da2b 100644 --- a/app/MindWork AI Studio/Pages/Information.razor.cs +++ b/app/MindWork AI Studio/Pages/Information.razor.cs @@ -35,7 +35,7 @@ public partial class Information : MSGComponentBase private static readonly MetaDataAttribute META_DATA = ASSEMBLY.GetCustomAttribute()!; private static readonly MetaDataArchitectureAttribute META_DATA_ARCH = ASSEMBLY.GetCustomAttribute()!; private static readonly MetaDataLibrariesAttribute META_DATA_LIBRARIES = ASSEMBLY.GetCustomAttribute()!; - private static readonly MetaDataDatabasesAttribute META_DATA_DATABASES = ASSEMBLY.GetCustomAttribute()!; + private static readonly MetaDataVectorStoreAttribute META_DATA_VECTOR_STORE = ASSEMBLY.GetCustomAttribute()!; private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(Information).Namespace, nameof(Information)); @@ -62,18 +62,18 @@ public partial class Information : MSGComponentBase private string VersionPdfium => $"{T("Used PDFium version")}: v{META_DATA_LIBRARIES.PdfiumVersion}"; - private string VersionDatabase + private string VersionVectorStore { get { - if (this.databaseClient is null) - return $"{T("Database")}: {T("checking availability")}"; + if (this.vectorStore is null) + return $"{T("Vector store")}: {T("checking availability")}"; - return this.databaseClient.Status switch + return this.vectorStore.Status switch { - DatabaseClientStatus.AVAILABLE => $"{T("Database version")}: {this.databaseClient.Name} v{META_DATA_DATABASES.DatabaseVersion}", - DatabaseClientStatus.STARTING => $"{T("Database")}: {this.databaseClient.Name} - {T("starting")}", - _ => $"{T("Database")}: {this.databaseClient.Name} - {T("not available")}" + DatabaseClientStatus.AVAILABLE => $"{T("Vector store version")}: {this.vectorStore.Name} v{META_DATA_VECTOR_STORE.VectorStoreVersion}", + DatabaseClientStatus.STARTING => $"{T("Vector store")}: {this.vectorStore.Name} - {T("starting")}", + _ => $"{T("Vector store")}: {this.vectorStore.Name} - {T("not available")}" }; } } @@ -85,7 +85,7 @@ private string VersionDatabase private bool showEnterpriseConfigDetails; - private bool showDatabaseDetails; + private bool showVectorStoreDetails; private List configPlugins = PluginFactory.AvailablePlugins .Where(x => x.Type is PluginType.CONFIGURATION) @@ -95,14 +95,13 @@ private string VersionDatabase private List enterpriseEnvironments = EnterpriseEnvironmentService.CURRENT_ENVIRONMENTS.ToList(); private List mandatoryInfoPanels = []; - - private sealed record DatabaseDisplayInfo(string Label, string Value); private sealed record MandatoryInfoPanelData(string HeaderText, string PluginName, DataMandatoryInfo Info, DataMandatoryInfoAcceptance? Acceptance); - - private readonly List databaseDisplayInfo = new(); - private DatabaseClient? databaseClient; - private CancellationTokenSource? databaseRefreshCancellationTokenSource; + + private sealed record VectorStoreDisplayInfo(string Label, string Value); + private readonly List vectorStoreDisplayInfo = new(); + private DatabaseClient? vectorStore; + private CancellationTokenSource? vectorStoreRefreshCancellationTokenSource; private bool HasAnyActiveEnvironment => this.enterpriseEnvironments.Any(e => e.IsActive); @@ -148,9 +147,9 @@ protected override async Task OnInitializedAsync() this.osUserName = await this.RustService.ReadUserName(); this.logPaths = await this.RustService.GetLogPaths(); - await this.RefreshDatabaseInfo(CancellationToken.None); - if (this.databaseClient?.Status is DatabaseClientStatus.STARTING) - this.StartShortDatabaseRefreshLoop(); + await this.RefreshVectorStoreInfo(CancellationToken.None); + if (this.vectorStore?.Status is DatabaseClientStatus.STARTING) + this.StartShortVectorStoreRefreshLoop(); // Determine the Pandoc version may take some time, so we start it here // without waiting for the result: @@ -249,22 +248,22 @@ private void ToggleEnterpriseConfigDetails() this.showEnterpriseConfigDetails = !this.showEnterpriseConfigDetails; } - private void ToggleDatabaseDetails() + private void ToggleVectorStoreDetails() { - this.showDatabaseDetails = !this.showDatabaseDetails; + this.showVectorStoreDetails = !this.showVectorStoreDetails; } - private async Task RefreshDatabaseInfo(CancellationToken cancellationToken) + private async Task RefreshVectorStoreInfo(CancellationToken cancellationToken) { var refreshedClient = await this.DatabaseClientProvider.RefreshClientAsync(DatabaseRole.VECTOR_STORE, cancellationToken); - this.databaseClient = refreshedClient; - this.databaseDisplayInfo.Clear(); + this.vectorStore = refreshedClient; + this.vectorStoreDisplayInfo.Clear(); try { await foreach (var (label, value) in refreshedClient.GetDisplayInfo().WithCancellation(cancellationToken)) { - this.databaseDisplayInfo.Add(new DatabaseDisplayInfo(label, value)); + this.vectorStoreDisplayInfo.Add(new VectorStoreDisplayInfo(label, value)); } } catch (OperationCanceledException) @@ -273,20 +272,20 @@ private async Task RefreshDatabaseInfo(CancellationToken cancellationToken) } catch (Exception e) { - this.databaseClient = new NoDatabaseClient(refreshedClient.Name, e.Message, DatabaseClientStatus.STARTING); - await foreach (var (label, value) in this.databaseClient.GetDisplayInfo().WithCancellation(cancellationToken)) + this.vectorStore = new NoDatabaseClient(refreshedClient.Name, e.Message, DatabaseClientStatus.STARTING); + await foreach (var (label, value) in this.vectorStore.GetDisplayInfo().WithCancellation(cancellationToken)) { - this.databaseDisplayInfo.Add(new DatabaseDisplayInfo(label, value)); + this.vectorStoreDisplayInfo.Add(new VectorStoreDisplayInfo(label, value)); } } } - private void StartShortDatabaseRefreshLoop() + private void StartShortVectorStoreRefreshLoop() { - this.databaseRefreshCancellationTokenSource?.Cancel(); - this.databaseRefreshCancellationTokenSource?.Dispose(); - this.databaseRefreshCancellationTokenSource = new CancellationTokenSource(); - var cancellationToken = this.databaseRefreshCancellationTokenSource.Token; + this.vectorStoreRefreshCancellationTokenSource?.Cancel(); + this.vectorStoreRefreshCancellationTokenSource?.Dispose(); + this.vectorStoreRefreshCancellationTokenSource = new CancellationTokenSource(); + var cancellationToken = this.vectorStoreRefreshCancellationTokenSource.Token; _ = Task.Run(async () => { @@ -298,11 +297,11 @@ private void StartShortDatabaseRefreshLoop() await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken); await this.InvokeAsync(async () => { - await this.RefreshDatabaseInfo(cancellationToken); + await this.RefreshVectorStoreInfo(cancellationToken); this.StateHasChanged(); }); - if (this.databaseClient?.Status is not DatabaseClientStatus.STARTING) + if (this.vectorStore?.Status is not DatabaseClientStatus.STARTING) return; } catch (OperationCanceledException) @@ -331,8 +330,8 @@ private bool IsManagedConfigurationIdMismatch(IAvailablePlugin plugin, Guid conf protected override void DisposeResources() { - this.databaseRefreshCancellationTokenSource?.Cancel(); - this.databaseRefreshCancellationTokenSource?.Dispose(); + this.vectorStoreRefreshCancellationTokenSource?.Cancel(); + this.vectorStoreRefreshCancellationTokenSource?.Dispose(); base.DisposeResources(); } diff --git a/app/MindWork AI Studio/Tools/Databases/DatabaseClientProvider.Qdrant.cs b/app/MindWork AI Studio/Tools/Databases/DatabaseClientProvider.Qdrant.cs new file mode 100644 index 000000000..f3676bb33 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Databases/DatabaseClientProvider.Qdrant.cs @@ -0,0 +1,80 @@ +using AIStudio.Tools.Databases.VectorStore; +using AIStudio.Tools.Rust; + +namespace AIStudio.Tools.Databases; + +public sealed partial class DatabaseClientProvider +{ + private async Task CreateQdrantClientAsync(CancellationToken cancellationToken) + { + var qdrantInfo = await rustService.GetQdrantInfo(cancellationToken); + if (qdrantInfo.Status is QdrantStatus.STARTING) + { + return this.CreateNoDatabaseClient( + "Qdrant", + "Qdrant is starting. Details will appear shortly.", + DatabaseClientStatus.STARTING); + } + + if (!qdrantInfo.IsAvailable || qdrantInfo.Status is QdrantStatus.UNAVAILABLE) + { + var reason = qdrantInfo.UnavailableReason ?? "unknown"; + this.logger.LogWarning("Qdrant is not available. Starting without vector database. Reason: '{Reason}'.", reason); + return this.CreateNoDatabaseClient("Qdrant", qdrantInfo.UnavailableReason, DatabaseClientStatus.UNAVAILABLE); + } + + if (!HasValidQdrantConnectionInfo(qdrantInfo, out var invalidReason)) + return this.CreateNoDatabaseClient("Qdrant", invalidReason, DatabaseClientStatus.UNAVAILABLE); + + var client = new QdrantClientImplementation("Qdrant", qdrantInfo.Path, qdrantInfo.PortHttp, qdrantInfo.PortGrpc, qdrantInfo.Fingerprint, qdrantInfo.ApiToken); + client.SetLogger(this.databaseClientLogger); + + try + { + await client.CheckAvailabilityAsync(); + return client; + } + catch (Exception e) + { + client.Dispose(); + this.logger.LogWarning(e, "Qdrant reported as available by Rust, but the health check failed."); + return this.CreateNoDatabaseClient("Qdrant", e.Message, DatabaseClientStatus.STARTING); + } + } + + private static bool HasValidQdrantConnectionInfo(QdrantInfo qdrantInfo, out string invalidReason) + { + if (qdrantInfo.Path == string.Empty) + { + invalidReason = "Failed to get the Qdrant path from Rust."; + return false; + } + + if (qdrantInfo.PortHttp == 0) + { + invalidReason = "Failed to get the Qdrant HTTP port from Rust."; + return false; + } + + if (qdrantInfo.PortGrpc == 0) + { + invalidReason = "Failed to get the Qdrant gRPC port from Rust."; + return false; + } + + if (qdrantInfo.Fingerprint == string.Empty) + { + invalidReason = "Failed to get the Qdrant fingerprint from Rust."; + return false; + } + + if (qdrantInfo.ApiToken == string.Empty) + { + invalidReason = "Failed to get the Qdrant API token from Rust."; + return false; + } + + invalidReason = string.Empty; + return true; + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Databases/DatabaseClientProvider.cs b/app/MindWork AI Studio/Tools/Databases/DatabaseClientProvider.cs index 4296ec533..55087784b 100644 --- a/app/MindWork AI Studio/Tools/Databases/DatabaseClientProvider.cs +++ b/app/MindWork AI Studio/Tools/Databases/DatabaseClientProvider.cs @@ -1,10 +1,9 @@ -using AIStudio.Tools.Databases.Qdrant; -using AIStudio.Tools.Rust; using AIStudio.Tools.Services; +using AIStudio.Tools.Databases.VectorStore; namespace AIStudio.Tools.Databases; -public sealed class DatabaseClientProvider(RustService rustService, ILoggerFactory loggerFactory) : IDisposable +public sealed partial class DatabaseClientProvider(RustService rustService, ILoggerFactory loggerFactory) : IDisposable { private readonly Dictionary clients = new(); private readonly Dictionary locks = new(); @@ -45,6 +44,18 @@ public async Task RefreshClientAsync(DatabaseRole databaseRole, } } + public async Task GetVectorStoreAsync(CancellationToken cancellationToken = default) + { + var client = await this.GetClientAsync(DatabaseRole.VECTOR_STORE, cancellationToken); + if (client is IVectorStoreClient vectorStore) + return vectorStore; + + return new NoVectorStoreClient( + client.Name, + "The configured database client does not support vector store operations.", + client.Status); + } + private DatabaseClient CacheIfAvailable(DatabaseRole databaseRole, DatabaseClient client) { if (!client.IsAvailable) @@ -84,79 +95,6 @@ private SemaphoreSlim GetLock(DatabaseRole databaseRole) _ => new NoDatabaseClient(databaseRole.ToString(), "The requested database role is not supported.") }; - private async Task CreateQdrantClientAsync(CancellationToken cancellationToken) - { - var qdrantInfo = await rustService.GetQdrantInfo(cancellationToken); - if (qdrantInfo.Status is QdrantStatus.STARTING) - { - return this.CreateNoDatabaseClient( - "Qdrant", - "Qdrant is starting. Details will appear shortly.", - DatabaseClientStatus.STARTING); - } - - if (!qdrantInfo.IsAvailable || qdrantInfo.Status is QdrantStatus.UNAVAILABLE) - { - var reason = qdrantInfo.UnavailableReason ?? "unknown"; - this.logger.LogWarning("Qdrant is not available. Starting without vector database. Reason: '{Reason}'.", reason); - return this.CreateNoDatabaseClient("Qdrant", qdrantInfo.UnavailableReason, DatabaseClientStatus.UNAVAILABLE); - } - - if (!HasValidQdrantConnectionInfo(qdrantInfo, out var invalidReason)) - return this.CreateNoDatabaseClient("Qdrant", invalidReason, DatabaseClientStatus.UNAVAILABLE); - - var client = new QdrantClientImplementation("Qdrant", qdrantInfo.Path, qdrantInfo.PortHttp, qdrantInfo.PortGrpc, qdrantInfo.Fingerprint, qdrantInfo.ApiToken); - client.SetLogger(this.databaseClientLogger); - - try - { - await client.CheckAvailabilityAsync(); - return client; - } - catch (Exception e) - { - client.Dispose(); - this.logger.LogWarning(e, "Qdrant reported as available by Rust, but the health check failed."); - return this.CreateNoDatabaseClient("Qdrant", e.Message, DatabaseClientStatus.STARTING); - } - } - - private static bool HasValidQdrantConnectionInfo(QdrantInfo qdrantInfo, out string invalidReason) - { - if (qdrantInfo.Path == string.Empty) - { - invalidReason = "Failed to get the Qdrant path from Rust."; - return false; - } - - if (qdrantInfo.PortHttp == 0) - { - invalidReason = "Failed to get the Qdrant HTTP port from Rust."; - return false; - } - - if (qdrantInfo.PortGrpc == 0) - { - invalidReason = "Failed to get the Qdrant gRPC port from Rust."; - return false; - } - - if (qdrantInfo.Fingerprint == string.Empty) - { - invalidReason = "Failed to get the Qdrant fingerprint from Rust."; - return false; - } - - if (qdrantInfo.ApiToken == string.Empty) - { - invalidReason = "Failed to get the Qdrant API token from Rust."; - return false; - } - - invalidReason = string.Empty; - return true; - } - private NoDatabaseClient CreateNoDatabaseClient(string name, string? unavailableReason, DatabaseClientStatus status) { var client = new NoDatabaseClient(name, unavailableReason, status); diff --git a/app/MindWork AI Studio/Tools/Databases/Qdrant/QdrantClientImplementation.cs b/app/MindWork AI Studio/Tools/Databases/Qdrant/QdrantClientImplementation.cs deleted file mode 100644 index b3a09e682..000000000 --- a/app/MindWork AI Studio/Tools/Databases/Qdrant/QdrantClientImplementation.cs +++ /dev/null @@ -1,73 +0,0 @@ -using Qdrant.Client; -using Qdrant.Client.Grpc; -using AIStudio.Tools.PluginSystem; - -namespace AIStudio.Tools.Databases.Qdrant; - -public class QdrantClientImplementation : DatabaseClient -{ - private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(QdrantClientImplementation).Namespace, nameof(QdrantClientImplementation)); - - private int HttpPort { get; } - - private int GrpcPort { get; } - - private QdrantClient GrpcClient { get; } - - private string Fingerprint { get; } - - private string ApiToken { get; } - - public QdrantClientImplementation(string name, string path, int httpPort, int grpcPort, string fingerprint, string apiToken): base(name, path) - { - this.HttpPort = httpPort; - this.GrpcPort = grpcPort; - this.Fingerprint = fingerprint; - this.ApiToken = apiToken; - this.GrpcClient = this.CreateQdrantClient(); - } - - public override string CacheKey => $"{this.Name}:{this.HttpPort}:{this.GrpcPort}:{this.Fingerprint}"; - - private const string IP_ADDRESS = "localhost"; - - private QdrantClient CreateQdrantClient() - { - var address = "https://" + IP_ADDRESS + ":" + this.GrpcPort; - var channel = QdrantChannel.ForAddress(address, new ClientConfiguration - { - ApiKey = this.ApiToken, - CertificateThumbprint = this.Fingerprint - }); - var grpcClient = new QdrantGrpcClient(channel); - return new QdrantClient(grpcClient); - } - - private async Task GetVersion() - { - var operation = await this.GrpcClient.HealthAsync(); - return $"v{operation.Version}"; - } - - public async Task CheckAvailabilityAsync() - { - await this.GrpcClient.HealthAsync(); - } - - private async Task GetCollectionsAmount() - { - var operation = await this.GrpcClient.ListCollectionsAsync(); - return operation.Count.ToString(); - } - - public override async IAsyncEnumerable<(string Label, string Value)> GetDisplayInfo() - { - yield return (TB("HTTP port"), this.HttpPort.ToString()); - yield return (TB("gRPC port"), this.GrpcPort.ToString()); - yield return (TB("Reported version"), await this.GetVersion()); - yield return (TB("Storage size"), $"{this.GetStorageSize()}"); - yield return (TB("Number of collections"), await this.GetCollectionsAmount()); - } - - public override void Dispose() => this.GrpcClient.Dispose(); -} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Databases/VectorStoragePoint.cs b/app/MindWork AI Studio/Tools/Databases/VectorStoragePoint.cs new file mode 100644 index 000000000..a614d6649 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Databases/VectorStoragePoint.cs @@ -0,0 +1,16 @@ +namespace AIStudio.Tools.Databases; + +public sealed record VectorStoragePoint( + string PointId, + IReadOnlyList Vector, + string DataSourceId, + string DataSourceName, + string DataSourceType, + string FilePath, + string FileName, + string RelativePath, + int ChunkIndex, + string Text, + string Fingerprint, + DateTime LastWriteUtc, + DateTime EmbeddedAtUtc); diff --git a/app/MindWork AI Studio/Tools/Databases/VectorStore/IVectorStoreClient.cs b/app/MindWork AI Studio/Tools/Databases/VectorStore/IVectorStoreClient.cs new file mode 100644 index 000000000..f1e96623d --- /dev/null +++ b/app/MindWork AI Studio/Tools/Databases/VectorStore/IVectorStoreClient.cs @@ -0,0 +1,20 @@ +namespace AIStudio.Tools.Databases.VectorStore; + +public interface IVectorStoreClient +{ + string Name { get; } + + DatabaseClientStatus Status { get; } + + bool IsAvailable { get; } + + IAsyncEnumerable<(string Label, string Value)> GetDisplayInfo(); + + Task EnsureVectorStoreExists(string storeName, int vectorSize, CancellationToken token); + + Task InsertEmbedding(string storeName, IReadOnlyList points, CancellationToken token); + + Task DeleteEmbeddingByFile(string storeName, string filePath, CancellationToken token); + + Task DeleteVectorStore(string storeName, CancellationToken token); +} diff --git a/app/MindWork AI Studio/Tools/Databases/VectorStore/NoVectorStoreClient.cs b/app/MindWork AI Studio/Tools/Databases/VectorStore/NoVectorStoreClient.cs new file mode 100644 index 000000000..6f9eaf878 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Databases/VectorStore/NoVectorStoreClient.cs @@ -0,0 +1,39 @@ +using AIStudio.Tools.PluginSystem; + +namespace AIStudio.Tools.Databases.VectorStore; + +public sealed class NoVectorStoreClient(string name, string? unavailableReason, DatabaseClientStatus status = DatabaseClientStatus.UNAVAILABLE) : IVectorStoreClient +{ + private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(NoVectorStoreClient).Namespace, nameof(NoVectorStoreClient)); + + public string Name => name; + + public DatabaseClientStatus Status => status; + + public bool IsAvailable => false; + + public async IAsyncEnumerable<(string Label, string Value)> GetDisplayInfo() + { + yield return (TB("Status"), TB("Unavailable")); + + if (!string.IsNullOrWhiteSpace(unavailableReason)) + yield return (TB("Reason"), unavailableReason); + + await Task.CompletedTask; + } + + public Task EnsureVectorStoreExists(string storeName, int vectorSize, CancellationToken token) => + Task.FromException(this.CreateUnavailableException()); + + public Task InsertEmbedding(string storeName, IReadOnlyList points, CancellationToken token) => + Task.FromException(this.CreateUnavailableException()); + + public Task DeleteEmbeddingByFile(string storeName, string filePath, CancellationToken token) => + Task.FromException(this.CreateUnavailableException()); + + public Task DeleteVectorStore(string storeName, CancellationToken token) => + Task.FromException(this.CreateUnavailableException()); + + private InvalidOperationException CreateUnavailableException() => + new(unavailableReason ?? "The vector store is not available."); +} diff --git a/app/MindWork AI Studio/Tools/Databases/VectorStore/QdrantClientImplementation.cs b/app/MindWork AI Studio/Tools/Databases/VectorStore/QdrantClientImplementation.cs new file mode 100644 index 000000000..97a1c600c --- /dev/null +++ b/app/MindWork AI Studio/Tools/Databases/VectorStore/QdrantClientImplementation.cs @@ -0,0 +1,142 @@ +using Qdrant.Client; +using Qdrant.Client.Grpc; +using Grpc.Core; +using AIStudio.Tools.PluginSystem; +using static Qdrant.Client.Grpc.Conditions; + +namespace AIStudio.Tools.Databases.VectorStore; + +public class QdrantClientImplementation : DatabaseClient, IVectorStoreClient +{ + private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(QdrantClientImplementation).Namespace, nameof(QdrantClientImplementation)); + + private int HttpPort { get; } + + private int GrpcPort { get; } + + private QdrantClient GrpcClient { get; } + + private string Fingerprint { get; } + + private string ApiToken { get; } + + public QdrantClientImplementation(string name, string path, int httpPort, int grpcPort, string fingerprint, string apiToken): base(name, path) + { + this.HttpPort = httpPort; + this.GrpcPort = grpcPort; + this.Fingerprint = fingerprint; + this.ApiToken = apiToken; + this.GrpcClient = this.CreateQdrantClient(); + } + + public override string CacheKey => $"{this.Name}:{this.HttpPort}:{this.GrpcPort}:{this.Fingerprint}"; + + private const string IP_ADDRESS = "localhost"; + + private QdrantClient CreateQdrantClient() + { + var address = "https://" + IP_ADDRESS + ":" + this.GrpcPort; + var channel = QdrantChannel.ForAddress(address, new ClientConfiguration + { + ApiKey = this.ApiToken, + CertificateThumbprint = this.Fingerprint + }); + var grpcClient = new QdrantGrpcClient(channel); + return new QdrantClient(grpcClient); + } + + private async Task GetVersion() + { + var operation = await this.GrpcClient.HealthAsync(); + return $"v{operation.Version}"; + } + + public async Task CheckAvailabilityAsync() + { + await this.GrpcClient.HealthAsync(); + } + + private async Task GetCollectionsAmount() + { + var operation = await this.GrpcClient.ListCollectionsAsync(); + return operation.Count.ToString(); + } + + public override async IAsyncEnumerable<(string Label, string Value)> GetDisplayInfo() + { + yield return (TB("HTTP port"), this.HttpPort.ToString()); + yield return (TB("gRPC port"), this.GrpcPort.ToString()); + yield return (TB("Reported version"), await this.GetVersion()); + yield return (TB("Storage size"), $"{this.GetStorageSize()}"); + yield return (TB("Number of collections"), await this.GetCollectionsAmount()); + } + + public async Task EnsureVectorStoreExists(string collectionName, int vectorSize, CancellationToken token) + { + var exists = await this.GrpcClient.CollectionExistsAsync(collectionName, token); + if (exists) + return; + + await this.GrpcClient.CreateCollectionAsync( + collectionName, + new VectorParams + { + Size = (ulong)vectorSize, + Distance = Distance.Cosine, + }, + cancellationToken: token); + } + + public Task InsertEmbedding(string collectionName, IReadOnlyList points, CancellationToken token) + { + var qdrantPoints = points.Select(point => new PointStruct + { + Id = Guid.Parse(point.PointId), + Vectors = point.Vector.ToArray(), + Payload = + { + ["data_source_id"] = point.DataSourceId, + ["data_source_name"] = point.DataSourceName, + ["data_source_type"] = point.DataSourceType, + ["file_path"] = point.FilePath, + ["file_name"] = point.FileName, + ["relative_path"] = point.RelativePath, + ["chunk_index"] = (long)point.ChunkIndex, + ["text"] = point.Text, + ["fingerprint"] = point.Fingerprint, + ["last_write_utc"] = point.LastWriteUtc.ToString("O"), + ["embedded_at_utc"] = point.EmbeddedAtUtc.ToString("O"), + } + }).ToList(); + + return this.GrpcClient.UpsertAsync(collectionName, qdrantPoints, true, null, null, token); + } + + public async Task DeleteEmbeddingByFile(string collectionName, string filePath, CancellationToken token) + { + try + { + await this.GrpcClient.DeleteAsync(collectionName, MatchKeyword("file_path", filePath), true, null, null, token); + } + catch (RpcException exception) when (exception.StatusCode is StatusCode.NotFound) + { + } + } + + public async Task DeleteVectorStore(string collectionName, CancellationToken token) + { + var exists = await this.GrpcClient.CollectionExistsAsync(collectionName, token); + if (!exists) + return; + + try + { + await this.GrpcClient.DeleteCollectionAsync(collectionName, cancellationToken: token); + } + catch (RpcException exception) when (exception.StatusCode is StatusCode.NotFound) + { + } + } + + public override void Dispose() => this.GrpcClient.Dispose(); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Metadata/MetaDataDatabasesAttribute.cs b/app/MindWork AI Studio/Tools/Metadata/MetaDataDatabasesAttribute.cs deleted file mode 100644 index 5ef6064b4..000000000 --- a/app/MindWork AI Studio/Tools/Metadata/MetaDataDatabasesAttribute.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace AIStudio.Tools.Metadata; - -public class MetaDataDatabasesAttribute(string databaseVersion) : Attribute -{ - public string DatabaseVersion => databaseVersion; -} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Metadata/MetaDataVectorStoreAttribute.cs b/app/MindWork AI Studio/Tools/Metadata/MetaDataVectorStoreAttribute.cs new file mode 100644 index 000000000..e3ba1b75f --- /dev/null +++ b/app/MindWork AI Studio/Tools/Metadata/MetaDataVectorStoreAttribute.cs @@ -0,0 +1,6 @@ +namespace AIStudio.Tools.Metadata; + +public class MetaDataVectorStoreAttribute(string vectorStoreVersion) : Attribute +{ + public string VectorStoreVersion => vectorStoreVersion; +} \ No newline at end of file