Background service for chunking and embedding data

app/MindWork AI Studio/Components/AttachDocuments.razor.cs

@@ -48,6 +48,9 @@ public partial class AttachDocuments : MSGComponentBase
     [Parameter]
     public bool UseSmallForm { get; set; }
 
+    [Parameter]
+    public FileType[]? AllowedFileTypes { get; set; }
+
     /// <summary>
     /// When true, validate media file types before attaching. Default is true. That means that
     /// the user cannot attach unsupported media file types when the provider or model does not

app/MindWork AI Studio/Components/ChatComponent.razor

@@ -34,7 +34,7 @@
     </ChildContent>
     <FooterContent>
         <MudElement Style="flex: 0 0 auto;">
-            <MudTextField
+            <UserPromptComponent
                 T="string"
                 @ref="@this.inputField"
                 @bind-Text="@this.UserInput"
@@ -50,8 +50,11 @@
                 Disabled="@this.IsInputForbidden()"
                 Immediate="@true"
                 OnKeyUp="@this.InputKeyEvent"
+                WhenTextChangedAsync="@(_ =>this.CalculateTokenCount())"
                 UserAttributes="@USER_INPUT_ATTRIBUTES"
                 Class="@this.UserInputClass"
+                DebounceTime="TimeSpan.FromSeconds(1)" 
+                HelperText="@this.TokenCountMessage"
                 Style="@this.UserInputStyle"/>
         </MudElement>
         <MudToolBar WrapContent="true" Gutters="@false" Class="border border-solid rounded" Style="border-color: lightgrey; gap: 2px;">

app/MindWork AI Studio/Components/ChatComponent.razor.cs

@@ -4,7 +4,7 @@
 using AIStudio.Settings;
 using AIStudio.Settings.DataModel;
 using AIStudio.Tools.AIJobs;
-
+using AIStudio.Tools.Services;
 using Microsoft.AspNetCore.Components;
 using Microsoft.AspNetCore.Components.Web;
 
@@ -48,6 +48,8 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable
     [Inject]
     private IDialogService DialogService { get; init; } = null!;
 
+    [Inject] 
+    private RustService RustService { get; init; } = null!;
     [Inject]
     private IJSRuntime JsRuntime { get; init; } = null!;
 
@@ -76,10 +78,14 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable
     private Guid loadedParameterWorkspaceId = Guid.Empty;
     private Guid foregroundChatId = Guid.Empty;
     private int workspaceHeaderSyncVersion;
+    private CancellationTokenSource? cancellationTokenSource;
+    private HashSet<FileAttachment> chatDocumentPaths = [];
+    private string tokenCount = "0";
+    private string TokenCountMessage => $"{this.T("Estimated amount of tokens:")} {this.tokenCount}";
 
     // Unfortunately, we need the input field reference to blur the focus away. Without
     // this, we cannot clear the input field.
-    private MudTextField<string> inputField = null!;
+    private UserPromptComponent<string> inputField = null!;
 
     /// <summary>
     /// Represents the user's input in the chat interface.
@@ -535,6 +541,9 @@ private async Task InputKeyEvent(KeyboardEventArgs keyEvent)
         // Was a modifier key pressed as well?
         var isModifier = keyEvent.AltKey || keyEvent.CtrlKey || keyEvent.MetaKey || keyEvent.ShiftKey;
 
+        if (isEnter)
+            await this.CalculateTokenCount();
+
         // Depending on the user's settings, might react to shortcuts:
         switch (this.SettingsManager.ConfigurationData.Chat.ShortcutSendBehavior)
         {
@@ -680,6 +689,7 @@ private async Task SendMessage(bool reuseLastUserPrompt = false)
         this.ComposerState.Clear();
 
         await this.inputField.BlurAsync();
+        this.tokenCount = "0";
 
         // Enable the stream state for the chat component:
         this.hasUnsavedChanges = true;
@@ -1028,6 +1038,35 @@ private void RestoreComposerFromTextBlock(ContentText textBlock)
         this.ComposerState.RestoreFromTextBlock(textBlock);
     }
 
+    private async Task CalculateTokenCount()
+    {
+        if (this.inputField.Value is null)
+        {
+            this.tokenCount = "0";
+            return;
+        }
+
+        var tokenizerResponse = await this.RustService.EnsureTokenizer(this.Provider.InstanceName, this.Provider.TokenizerPath);
+        if (tokenizerResponse is null)
+            return;
+        if (!tokenizerResponse.Value.Success)
+        {
+            this.Logger.LogWarning($"Failed to initialize the tokenizer for the provider: status='{tokenizerResponse.Value.Status}', reason='{tokenizerResponse.Value.Message}'");
+            return;
+        }
+
+        var response = await this.RustService.GetTokenCount(this.inputField.Value);
+        if (response is null)
+            return;
+        if (!response.Value.Success)
+        {
+            this.Logger.LogWarning($"Failed to calculate token count: status='{response.Value.Status}', reason='{response.Value.Message}'");
+            return;
+        }
+        this.tokenCount = response.Value.TokenCount.ToString();
+        this.StateHasChanged();
+    }
+
     #region Overrides of MSGComponentBase
 
     protected override async Task ProcessIncomingMessage<T>(ComponentBase? sendingComponent, Event triggeredEvent, T? data) where T : default

app/MindWork AI Studio/Components/SelectFile.razor

@@ -1,19 +1,42 @@
-@inherits MSGComponentBase
+@inherits MSGComponentBase
 
-<MudStack Row="@true" Spacing="3" Class="mb-3" StretchItems="StretchItems.None" AlignItems="AlignItems.Center">
+<style>
+    .select-file-button-wrapper {
+        height: 56px;
+        min-height: 56px;
+        max-height: 56px;
+        display: flex;
+        align-items: center;
+        align-self: flex-start;
+        flex-shrink: 0;
+        margin-top: 8px;
+    }
+</style>
+
+<MudStack Row="@true" Spacing="3" Class="mb-3" StretchItems="StretchItems.None" AlignItems="AlignItems.Start">
     <MudTextField
         T="string"
         Text="@this.File"
         Label="@this.Label"
-        ReadOnly="@true"
+        ReadOnly="@(!this.IsClearable)"
         Validation="@this.Validation"
         Adornment="Adornment.Start"
         AdornmentIcon="@Icons.Material.Filled.AttachFile"
         UserAttributes="@SPELLCHECK_ATTRIBUTES"
         Variant="Variant.Outlined"
+        Clearable="@this.IsClearable"
+        Error="@this.Error"
+        ErrorText="@this.ErrorText"
+        OnClearButtonClick="@this.OnClear"
     />
 
-    <MudButton StartIcon="@Icons.Material.Filled.FolderOpen" Variant="Variant.Outlined" Color="Color.Primary" Disabled="this.Disabled" OnClick="@this.OpenFileDialog">
-        @T("Choose File")
-    </MudButton>
-</MudStack>
+    <div class="select-file-button-wrapper">
+        <MudButton StartIcon="@Icons.Material.Filled.FolderOpen"
+                   Variant="Variant.Outlined"
+                   Color="Color.Primary"
+                   Disabled="@this.Disabled"
+                   OnClick="@this.OpenFileDialog">
+            @T("Choose File")
+        </MudButton>
+    </div>
+</MudStack>

app/MindWork AI Studio/Components/SelectFile.razor.cs

@@ -2,6 +2,7 @@
 using AIStudio.Tools.Services;
 
 using Microsoft.AspNetCore.Components;
+using Microsoft.AspNetCore.Components.Web;
 
 namespace AIStudio.Components;
 
@@ -27,7 +28,19 @@ public partial class SelectFile : MSGComponentBase
 
     [Parameter]
     public Func<string, string?> Validation { get; set; } = _ => null;
-
+
+    [Parameter]
+    public bool IsClearable { get; set; } = false;
+
+    [Parameter]
+    public bool Error { get; set; } = false;
+
+    [Parameter]
+    public string ErrorText { get; set; } = string.Empty;
+
+    [Parameter]
+    public Func<MouseEventArgs, Task> OnClear { get; set; } = _ => Task.CompletedTask;
+
     [Inject]
     public RustService RustService { get; set; } = null!;
 
@@ -52,7 +65,7 @@ private void InternalFileChanged(string file)
         this.File = file;
         this.FileChanged.InvokeAsync(file);
     }
-
+    
     private async Task OpenFileDialog()
     {
         var response = await this.RustService.SelectFile(this.FileDialogTitle, this.Filter, string.IsNullOrWhiteSpace(this.File) ? null : this.File);

app/MindWork AI Studio/Components/Settings/SettingsPanelEmbeddings.razor.cs

@@ -2,6 +2,8 @@
 using AIStudio.Dialogs;
 using AIStudio.Provider;
 using AIStudio.Settings;
+using AIStudio.Tools.Services;
+using AIStudio.Tools.Rust;
 
 using Microsoft.AspNetCore.Components;
 
@@ -11,6 +13,9 @@ namespace AIStudio.Components.Settings;
 
 public partial class SettingsPanelEmbeddings : SettingsPanelProviderBase
 {
+    [Inject]
+    private DataSourceEmbeddingService DataSourceEmbeddingService { get; init; } = null!;
+
     [Parameter]
     public List<ConfigurationSelectData<string>> AvailableEmbeddingProviders { get; set; } = new();
 
@@ -57,6 +62,7 @@ private async Task AddEmbeddingProvider()
         await this.UpdateEmbeddingProviders();
 
         await this.SettingsManager.StoreSettings();
+        await this.DataSourceEmbeddingService.QueueAllInternalDataSourcesAsync();
         await this.MessageBus.SendMessage<bool>(this, Event.CONFIGURATION_CHANGED);
     }
 
@@ -73,6 +79,7 @@ private async Task EditEmbeddingProvider(EmbeddingProvider embeddingProvider)
             { x => x.IsSelfHosted, embeddingProvider.IsSelfHosted },
             { x => x.IsEditing, true },
             { x => x.DataHost, embeddingProvider.Host },
+            { x => x.DataTokenizerPath, embeddingProvider.TokenizerPath },
         };
 
         var dialogReference = await this.DialogService.ShowAsync<EmbeddingProviderDialog>(T("Edit Embedding Provider"), dialogParameters, DialogOptions.FULLSCREEN);
@@ -91,6 +98,7 @@ private async Task EditEmbeddingProvider(EmbeddingProvider embeddingProvider)
         await this.UpdateEmbeddingProviders();
 
         await this.SettingsManager.StoreSettings();
+        await this.DataSourceEmbeddingService.QueueAllInternalDataSourcesAsync();
         await this.MessageBus.SendMessage<bool>(this, Event.CONFIGURATION_CHANGED);
     }
 
@@ -107,16 +115,45 @@ private async Task DeleteEmbeddingProvider(EmbeddingProvider provider)
             return;
 
         var deleteSecretResponse = await this.RustService.DeleteAPIKey(provider, SecretStoreType.EMBEDDING_PROVIDER);
-        if(deleteSecretResponse.Success)
+        var deleteTokenizerResponse = await this.RustService.DeleteTokenizer(TokenizerModelId.ForEmbeddingProvider(provider));
+        if(deleteSecretResponse.Success && deleteTokenizerResponse.Success)
+        {
+            this.SettingsManager.ConfigurationData.EmbeddingProviders.Remove(provider);
+            await this.SettingsManager.StoreSettings();
+        }
+        else
         {
+            var issueDialogParameters = new DialogParameters<ConfirmDialog>
+            {
+                { x => x.Message, string.Format(T("Couldn't delete the embedding provider '{0}'. The issue: {1}. We can ignore this issue and delete the embedding provider anyway. Do you want to ignore it and delete this embedding provider?"), provider.Name, BuildDeleteIssue(deleteSecretResponse, deleteTokenizerResponse)) },
+            };
+
+            var issueDialogReference = await this.DialogService.ShowAsync<ConfirmDialog>(T("Delete Embedding Provider"), issueDialogParameters, DialogOptions.FULLSCREEN);
+            var issueDialogResult = await issueDialogReference.Result;
+            if (issueDialogResult is null || issueDialogResult.Canceled)
+                return;
+
             this.SettingsManager.ConfigurationData.EmbeddingProviders.Remove(provider);
             await this.SettingsManager.StoreSettings();
         }
 
         await this.UpdateEmbeddingProviders();
+        await this.DataSourceEmbeddingService.QueueAllInternalDataSourcesAsync();
         await this.MessageBus.SendMessage<bool>(this, Event.CONFIGURATION_CHANGED);
     }
 
+    private static string BuildDeleteIssue(DeleteSecretResponse deleteSecretResponse, TokenizerResponse deleteTokenizerResponse)
+    {
+        var issues = new List<string>();
+        if (!deleteSecretResponse.Success)
+            issues.Add(deleteSecretResponse.Issue);
+
+        if (!deleteTokenizerResponse.Success)
+            issues.Add(deleteTokenizerResponse.Message);
+
+        return string.Join(" | ", issues);
+    }
+
     private async Task ExportEmbeddingProvider(EmbeddingProvider provider)
     {
         if (!this.SettingsManager.ConfigurationData.App.ShowAdminSettings)
@@ -156,7 +193,7 @@ private async Task TestEmbeddingProvider(EmbeddingProvider provider)
             return;
 
         var embeddingProvider = provider.CreateProvider();
-        var embeddings = await embeddingProvider.EmbedTextAsync(provider.Model, this.SettingsManager, default, new List<string> { inputText });
+        var embeddings = await embeddingProvider.EmbedTextAsync(provider.Model, this.SettingsManager, CancellationToken.None, inputText);
 
         if (embeddings.Count == 0)
         {

app/MindWork AI Studio/Components/Settings/SettingsPanelProviders.razor.cs

@@ -3,6 +3,8 @@
 using AIStudio.Dialogs;
 using AIStudio.Provider;
 using AIStudio.Settings;
+using AIStudio.Tools.Rust;
+using AIStudio.Tools.Services;
 
 using Microsoft.AspNetCore.Components;
 
@@ -73,6 +75,7 @@ private async Task EditLLMProvider(AIStudio.Settings.Provider provider)
             { x => x.DataHost, provider.Host },
             { x => x.HFInferenceProviderId, provider.HFInferenceProvider },
             { x => x.AdditionalJsonApiParameters, provider.AdditionalJsonApiParameters },
+            { x => x.DataTokenizerPath, provider.TokenizerPath },
         };
 
         var dialogReference = await this.DialogService.ShowAsync<ProviderDialog>(T("Edit LLM Provider"), dialogParameters, DialogOptions.FULLSCREEN);
@@ -108,7 +111,8 @@ private async Task DeleteLLMProvider(AIStudio.Settings.Provider provider)
             return;
 
         var deleteSecretResponse = await this.RustService.DeleteAPIKey(provider, SecretStoreType.LLM_PROVIDER);
-        if(deleteSecretResponse.Success)
+        var deleteTokenizerResponse = await this.RustService.DeleteTokenizer(TokenizerModelId.ForProvider(provider));
+        if(deleteSecretResponse.Success && deleteTokenizerResponse.Success)
         {
             this.SettingsManager.ConfigurationData.Providers.Remove(provider);
             await this.SettingsManager.StoreSettings();
@@ -117,7 +121,7 @@ private async Task DeleteLLMProvider(AIStudio.Settings.Provider provider)
         {
             var issueDialogParameters = new DialogParameters<ConfirmDialog>
             {
-                { x => x.Message, string.Format(T("Couldn't delete the provider '{0}'. The issue: {1}. We can ignore this issue and delete the provider anyway. Do you want to ignore it and delete this provider?"), provider.InstanceName, deleteSecretResponse.Issue) },
+                { x => x.Message, string.Format(T("Couldn't delete the provider '{0}'. The issue: {1}. We can ignore this issue and delete the provider anyway. Do you want to ignore it and delete this provider?"), provider.InstanceName, BuildDeleteIssue(deleteSecretResponse, deleteTokenizerResponse)) },
             };
 
             var issueDialogReference = await this.DialogService.ShowAsync<ConfirmDialog>(T("Delete LLM Provider"), issueDialogParameters, DialogOptions.FULLSCREEN);
@@ -134,6 +138,18 @@ private async Task DeleteLLMProvider(AIStudio.Settings.Provider provider)
         await this.MessageBus.SendMessage<bool>(this, Event.CONFIGURATION_CHANGED);
     }
 
+    private static string BuildDeleteIssue(DeleteSecretResponse deleteSecretResponse, TokenizerResponse deleteTokenizerResponse)
+    {
+        var issues = new List<string>();
+        if (!deleteSecretResponse.Success)
+            issues.Add(deleteSecretResponse.Issue);
+
+        if (!deleteTokenizerResponse.Success)
+            issues.Add(deleteTokenizerResponse.Message);
+
+        return string.Join(" | ", issues);
+    }
+
     private async Task ExportLLMProvider(AIStudio.Settings.Provider provider)
     {
         if (!this.SettingsManager.ConfigurationData.App.ShowAdminSettings)