fix(vram): persist remote probe metadata

core/application/startup.go

@@ -443,13 +443,20 @@ func New(opts ...config.AppOption) (*Application, error) {
 	// Wire gallery generation counter into VRAM caches so they invalidate
 	// when gallery data refreshes instead of using a fixed TTL.
 	vram.SetGalleryGenerationFunc(gallery.GalleryGeneration)
+	if options.AutoloadGalleries {
+		if options.VRAMPersistentCache {
+			// Remote GGUF probes can transfer substantial metadata. Keep successful
+			// results across restarts so the startup warmer does not repeat that work.
+			vram.ConfigurePersistentCache(filepath.Join(options.SystemState.Model.ModelsPath, "..", "cache", "vram"), 24*time.Hour)
+		}
 
-	// Fill those caches ahead of the first visitor. An estimate for an entry
-	// nobody has asked about yet costs a remote probe of its weight files, and
-	// the model gallery asks for one per row, so without this the first page
-	// spends seconds filling in its own sizes while somebody watches it.
-	// Non-blocking, and bounded: see DefaultEstimateWarmConfig.
-	gallery.WarmEstimateCache(options.Context, options.Galleries, options.SystemState, gallery.EstimateWarmConfigFromEnv())
+		// Fill those caches ahead of the first visitor. An estimate for an entry
+		// nobody has asked about yet costs a remote probe of its weight files, and
+		// the model gallery asks for one per row, so without this the first page
+		// spends seconds filling in its own sizes while somebody watches it.
+		// Non-blocking, and bounded: see DefaultEstimateWarmConfig.
+		gallery.WarmEstimateCache(options.Context, options.Galleries, options.SystemState, gallery.EstimateWarmConfigFromEnv())
+	}
 
 	if options.ConfigFile != "" {
 		if err := application.ModelConfigLoader().LoadMultipleModelConfigsSingleFile(options.ConfigFile, configLoaderOpts...); err != nil {

core/cli/run.go

@@ -53,6 +53,7 @@ type RunCMD struct {
 	BackendGalleries          string   `env:"LOCALAI_BACKEND_GALLERIES,BACKEND_GALLERIES" help:"JSON list of backend galleries" group:"backends" default:"${backends}"`
 	Galleries                 string   `env:"LOCALAI_GALLERIES,GALLERIES" help:"JSON list of galleries" group:"models" default:"${galleries}"`
 	AutoloadGalleries         bool     `env:"LOCALAI_AUTOLOAD_GALLERIES,AUTOLOAD_GALLERIES" group:"models" default:"true"`
+	VRAMPersistentCache       bool     `env:"LOCALAI_VRAM_PERSISTENT_CACHE,VRAM_PERSISTENT_CACHE" group:"models" default:"true" help:"Persist successful remote VRAM metadata probes across restarts"`
 	AutoloadBackendGalleries  bool     `env:"LOCALAI_AUTOLOAD_BACKEND_GALLERIES,AUTOLOAD_BACKEND_GALLERIES" group:"backends" default:"true"`
 	BackendImagesReleaseTag   string   `env:"LOCALAI_BACKEND_IMAGES_RELEASE_TAG,BACKEND_IMAGES_RELEASE_TAG" help:"Fallback release tag for backend images" group:"backends" default:"latest"`
 	BackendImagesBranchTag    string   `env:"LOCALAI_BACKEND_IMAGES_BRANCH_TAG,BACKEND_IMAGES_BRANCH_TAG" help:"Fallback branch tag for backend images" group:"backends" default:"master"`
@@ -300,6 +301,7 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error {
 		config.WithF16(r.F16),
 		config.WithStringGalleries(r.Galleries),
 		config.WithBackendGalleries(r.BackendGalleries),
+		config.WithVRAMPersistentCache(r.VRAMPersistentCache),
 		config.WithCors(r.CORS),
 		config.WithCorsAllowOrigins(r.CORSAllowOrigins),
 		config.WithDisableCSRF(r.DisableCSRF),

core/config/application_config.go

@@ -124,6 +124,7 @@ type ApplicationConfig struct {
 	ExternalGRPCBackends map[string]string
 
 	AutoloadGalleries, AutoloadBackendGalleries bool
+	VRAMPersistentCache                         bool
 	AutoUpgradeBackends                         bool
 	PreferDevelopmentBackends                   bool
 
@@ -281,6 +282,7 @@ func NewApplicationConfig(o ...AppOption) *ApplicationConfig {
 		// toggle can still turn it off (a persisted false wins - see
 		// loadRuntimeSettingsFromFile).
 		EnableBackendLogging:        true,
+		VRAMPersistentCache:         true,
 		ArtifactDownloadConcurrency: modelartifacts.DefaultDownloadConcurrency,
 		AgentJobRetentionDays:       30,               // Default: 30 days
 		LRUEvictionMaxRetries:       30,               // Default: 30 retries
@@ -620,6 +622,10 @@ func WithAutoUpgradeBackends(v bool) AppOption {
 	return func(o *ApplicationConfig) { o.AutoUpgradeBackends = v }
 }
 
+func WithVRAMPersistentCache(v bool) AppOption {
+	return func(o *ApplicationConfig) { o.VRAMPersistentCache = v }
+}
+
 func WithRequireBackendIntegrity(v bool) AppOption {
 	return func(o *ApplicationConfig) { o.RequireBackendIntegrity = v }
 }

core/config/application_config_test.go

@@ -1,6 +1,7 @@
 package config
 
 import (
+	"encoding/json"
 	"time"
 
 	. "github.com/onsi/ginkgo/v2"
@@ -9,6 +10,15 @@ import (
 
 var _ = Describe("ApplicationConfig RuntimeSettings Conversion", func() {
 	Describe("ToRuntimeSettings", func() {
+		It("includes the persistent VRAM cache toggle", func() {
+			encoded, err := json.Marshal(NewApplicationConfig().ToRuntimeSettings())
+			Expect(err).NotTo(HaveOccurred())
+
+			var settings map[string]any
+			Expect(json.Unmarshal(encoded, &settings)).To(Succeed())
+			Expect(settings).To(HaveKeyWithValue("vram_persistent_cache", true))
+		})
+
 		It("should convert all fields correctly", func() {
 			appConfig := &ApplicationConfig{
 				WatchDog:                 true,

core/config/runtime_settings.go

@@ -59,6 +59,7 @@ type RuntimeSettings struct {
 	BackendGalleries         *[]Gallery `json:"backend_galleries,omitempty"`
 	AutoloadGalleries        *bool      `json:"autoload_galleries,omitempty"`
 	AutoloadBackendGalleries *bool      `json:"autoload_backend_galleries,omitempty"`
+	VRAMPersistentCache      *bool      `json:"vram_persistent_cache,omitempty"`
 
 	// API keys - No omitempty as we need to save empty arrays to clear keys
 	ApiKeys *[]string `json:"api_keys"`

core/config/runtime_settings_registry.go

@@ -328,6 +328,10 @@ var runtimeSettingsFields = []fieldSpec{
 		func(s *RuntimeSettings) **bool { return &s.AutoloadBackendGalleries },
 		func(o *ApplicationConfig) bool { return o.AutoloadBackendGalleries },
 		func(o *ApplicationConfig, v bool) { o.AutoloadBackendGalleries = v }),
+	field("vram_persistent_cache",
+		func(s *RuntimeSettings) **bool { return &s.VRAMPersistentCache },
+		func(o *ApplicationConfig) bool { return o.VRAMPersistentCache },
+		func(o *ApplicationConfig, v bool) { o.VRAMPersistentCache = v }),
 
 	// API keys: echoed for the UI, but the apply loops never touch them.
 	// The settings endpoint and the file watcher own the env+runtime merge

core/config/runtime_settings_startup.go

@@ -45,6 +45,7 @@ func DefaultRuntimeBaseline() *ApplicationConfig {
 	o.BackendGalleries = mustGalleries(DefaultBackendGalleriesJSON)
 	o.AutoloadGalleries = true
 	o.AutoloadBackendGalleries = true
+	o.VRAMPersistentCache = true
 	// core/cli/run.go injects WithMemoryReclaimer(enabled, threshold)
 	// unconditionally, so the kong threshold default (0.95) reaches the
 	// config even when the reclaimer flag is off - this overlay must match

core/gallery/gallery.go

@@ -16,6 +16,7 @@ import (
 	"github.com/mudler/LocalAI/pkg/downloader"
 	"github.com/mudler/LocalAI/pkg/system"
 	"github.com/mudler/LocalAI/pkg/utils"
+	"github.com/mudler/LocalAI/pkg/vram"
 	"github.com/mudler/LocalAI/pkg/xsync"
 	"github.com/mudler/xlog"
 
@@ -457,6 +458,9 @@ func triggerGalleryRefresh(galleries []config.Gallery, systemState *system.Syste
 			galleryGeneration.Add(1)
 		}
 		availableModelsMu.Unlock()
+		if changed {
+			vram.InvalidatePersistentCache()
+		}
 	}()
 }
 

core/http/endpoints/localai/settings.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"io"
 	"net/http"
+	"path/filepath"
 	"time"
 
 	"github.com/labstack/echo/v4"
@@ -12,6 +13,7 @@ import (
 	"github.com/mudler/LocalAI/core/http/endpoints/openresponses"
 	"github.com/mudler/LocalAI/core/p2p"
 	"github.com/mudler/LocalAI/core/schema"
+	"github.com/mudler/LocalAI/pkg/vram"
 	"github.com/mudler/LocalAI/pkg/vrambudget"
 	"github.com/mudler/xlog"
 )
@@ -185,6 +187,13 @@ func UpdateSettingsEndpoint(app *application.Application) echo.HandlerFunc {
 
 		// Apply settings using centralized method
 		watchdogChanged := appConfig.ApplyRuntimeSettings(&settings)
+		if settings.VRAMPersistentCache != nil || settings.AutoloadGalleries != nil {
+			if appConfig.VRAMPersistentCache && appConfig.AutoloadGalleries {
+				vram.ConfigurePersistentCache(filepath.Join(appConfig.SystemState.Model.ModelsPath, "..", "cache", "vram"), 24*time.Hour)
+			} else {
+				vram.DisablePersistentCache()
+			}
+		}
 
 		// Handle API keys specially (merge with startup keys)
 		if settings.ApiKeys != nil {

core/http/react-ui/e2e/settings-backend-logging.spec.js

@@ -18,6 +18,34 @@ test.describe('Settings - Backend Logging', () => {
     await expect(input).toHaveValue('4')
   })
 
+  test('persistent VRAM cache can be toggled', async ({ page }) => {
+    const row = page.locator('.form-row', { hasText: 'Persist remote VRAM estimates' })
+    await expect(row).toBeVisible()
+
+    const checkbox = row.locator('input[type="checkbox"]')
+    const wasChecked = await checkbox.isChecked()
+    await checkbox.locator('..').click()
+    if (wasChecked) {
+      await expect(checkbox).not.toBeChecked()
+    } else {
+      await expect(checkbox).toBeChecked()
+    }
+  })
+
+  test('gallery startup loading and pre-warming can be toggled together', async ({ page }) => {
+    const row = page.locator('.form-row', { hasText: 'Load and pre-warm galleries on boot' })
+    await expect(row).toBeVisible()
+
+    const checkbox = row.locator('input[type="checkbox"]')
+    const wasChecked = await checkbox.isChecked()
+    await checkbox.locator('..').click()
+    if (wasChecked) {
+      await expect(checkbox).not.toBeChecked()
+    } else {
+      await expect(checkbox).toBeChecked()
+    }
+  })
+
   test('backend logging toggle can be toggled', async ({ page }) => {
     // Find the checkbox associated with backend logging
     const section = page.locator('div', { has: page.locator('text=Enable Backend Logging') })

core/http/react-ui/src/pages/Settings.jsx

@@ -482,12 +482,15 @@ export default function Settings() {
               <i className="fas fa-images text-accent" /> Galleries
             </h3>
             <div className="card">
-              <SettingRow label="Autoload Galleries" description="Automatically load model galleries on startup">
+              <SettingRow label="Load and pre-warm galleries on boot" description="Load model galleries and pre-warm their remote size and VRAM estimates when LocalAI starts">
                 <Toggle checked={settings.autoload_galleries} onChange={(v) => update('autoload_galleries', v)} />
               </SettingRow>
               <SettingRow label="Autoload Backend Galleries" description="Automatically load backend galleries on startup">
                 <Toggle checked={settings.autoload_backend_galleries} onChange={(v) => update('autoload_backend_galleries', v)} />
               </SettingRow>
+              <SettingRow label="Persist remote VRAM estimates" description="Reuse successful remote model metadata probes across restarts; disabled when gallery autoload is off">
+                <Toggle checked={settings.vram_persistent_cache} onChange={(v) => update('vram_persistent_cache', v)} />
+              </SettingRow>
               <div className="mt-sm">
                 <label className="form-label">Model Galleries (JSON)</label>
                 <textarea

docs/content/advanced/vram-management.md

@@ -460,8 +460,16 @@ context lengths, so you can see whether something will run before installing it.
 Working that out means reading the metadata of a model's weight files, which for
 a model you have not installed is a request to the host that serves them. It
 takes a second or two the first time, and the gallery needs one per row. LocalAI
-caches the result, and warms that cache in the background at startup so the
-gallery reads instantly rather than filling in its own numbers while you watch.
+caches successful remote probes for 24 hours under the LocalAI data directory,
+and warms that cache in the background at startup so the gallery reads instantly
+rather than filling in its own numbers while you watch. The on-disk cache is
+reused after a restart, so frequent restarts do not download the same metadata
+again. Local model files are always inspected directly. The cache keeps at most
+4,096 entries and removes the oldest entries when it reaches that limit.
+Disable **Persist remote VRAM estimates** under **Settings > Galleries**, or set
+`LOCALAI_VRAM_PERSISTENT_CACHE=false`, to keep estimates in memory only. Setting
+`LOCALAI_AUTOLOAD_GALLERIES=false` also disables the startup warmer and the
+persistent cache.
 
 The same warm-up also describes each entry's **variants** - the alternative
 builds of the same weights that the picker offers - because that costs the same

docs/content/features/runtime-settings.md

@@ -82,7 +82,7 @@ Manage model and backend galleries:
 
 - **Model Galleries**: JSON array of gallery objects with `url` and `name` fields, plus an optional `mirrors` list of fallback URLs (see [Gallery mirrors]({{%relref "features/model-gallery#gallery-mirrors" %}}))
 - **Backend Galleries**: JSON array of backend gallery objects, which accept the same `mirrors` key
-- **Autoload Galleries**: Automatically load model galleries on startup
+- **Load and pre-warm galleries on boot**: Load model galleries and pre-warm their remote size and VRAM estimates when LocalAI starts. Disable this setting to skip both startup operations.
 - **Autoload Backend Galleries**: Automatically load backend galleries on startup
 
 ### Agent Pool Settings
@@ -164,6 +164,7 @@ The `runtime_settings.json` file follows this structure:
   ],
   "autoload_galleries": true,
   "autoload_backend_galleries": true,
+  "vram_persistent_cache": true,
   "api_keys": []
 }
 ```

docs/content/reference/cli-reference.md

@@ -61,6 +61,7 @@ For more information on VRAM management, see [VRAM and Memory Management]({{%rel
 |-----------|---------|-------------|----------------------|
 | `--galleries` | | JSON list of galleries | `$LOCALAI_GALLERIES`, `$GALLERIES` |
 | `--autoload-galleries` | `true` | Automatically load galleries on startup | `$LOCALAI_AUTOLOAD_GALLERIES`, `$AUTOLOAD_GALLERIES` |
+| `--vram-persistent-cache` | `true` | Persist successful remote VRAM metadata probes across restarts | `$LOCALAI_VRAM_PERSISTENT_CACHE`, `$VRAM_PERSISTENT_CACHE` |
 | `--preload-models` | | A list of models to apply in JSON at start | `$LOCALAI_PRELOAD_MODELS`, `$PRELOAD_MODELS` |
 | `--models` | | A list of model configuration URLs to load | `$LOCALAI_MODELS`, `$MODELS` |
 | `--preload-models-config` | | A list of models to apply at startup. Path to a YAML config file | `$LOCALAI_PRELOAD_MODELS_CONFIG`, `$PRELOAD_MODELS_CONFIG` |