diff --git a/core/application/startup.go b/core/application/startup.go index 19966ecb23f8..0d69763b486e 100644 --- a/core/application/startup.go +++ b/core/application/startup.go @@ -136,6 +136,8 @@ func New(opts ...config.AppOption) (*Application, error) { loadRuntimeSettingsFromFile(options) } + application.ModelLoader().SetBackendLoggingEnabled(options.EnableBackendLogging) + // turn off any process that was started by GRPC if the context is canceled go func() { <-options.Context.Done() @@ -382,6 +384,12 @@ func loadRuntimeSettingsFromFile(options *config.ApplicationConfig) { } } + if settings.EnableBackendLogging != nil { + if !options.EnableBackendLogging { + options.EnableBackendLogging = *settings.EnableBackendLogging + } + } + xlog.Debug("Runtime settings loaded from runtime_settings.json") } diff --git a/core/backend/detection.go b/core/backend/detection.go index 1b1991824145..c7f866862b82 100644 --- a/core/backend/detection.go +++ b/core/backend/detection.go @@ -3,8 +3,10 @@ package backend import ( "context" "fmt" + "time" "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/trace" "github.com/mudler/LocalAI/pkg/grpc/proto" "github.com/mudler/LocalAI/pkg/model" ) @@ -18,6 +20,7 @@ func Detection( opts := ModelOptions(modelConfig, appConfig) detectionModel, err := loader.Load(opts...) if err != nil { + recordModelLoadFailure(appConfig, modelConfig.Name, modelConfig.Backend, err, nil) return nil, err } @@ -25,9 +28,35 @@ func Detection( return nil, fmt.Errorf("could not load detection model") } + var startTime time.Time + if appConfig.EnableTracing { + trace.InitBackendTracingIfEnabled(appConfig.TracingMaxItems) + startTime = time.Now() + } + res, err := detectionModel.Detect(context.Background(), &proto.DetectOptions{ Src: sourceFile, }) + if appConfig.EnableTracing { + errStr := "" + if err != nil { + errStr = err.Error() + } + + trace.RecordBackendTrace(trace.BackendTrace{ + Timestamp: startTime, + Duration: time.Since(startTime), + Type: trace.BackendTraceDetection, + ModelName: modelConfig.Name, + Backend: modelConfig.Backend, + Summary: trace.TruncateString(sourceFile, 200), + Error: errStr, + Data: map[string]any{ + "source_file": sourceFile, + }, + }) + } + return res, err } diff --git a/core/backend/embeddings.go b/core/backend/embeddings.go index 2ba5049f240d..382f8f3583bc 100644 --- a/core/backend/embeddings.go +++ b/core/backend/embeddings.go @@ -17,6 +17,7 @@ func ModelEmbedding(s string, tokens []int, loader *model.ModelLoader, modelConf inferenceModel, err := loader.Load(opts...) if err != nil { + recordModelLoadFailure(appConfig, modelConfig.Name, modelConfig.Backend, err, nil) return nil, err } diff --git a/core/backend/image.go b/core/backend/image.go index 89ea14cb190e..44ca010c218c 100644 --- a/core/backend/image.go +++ b/core/backend/image.go @@ -17,6 +17,7 @@ func ImageGeneration(height, width, step, seed int, positive_prompt, negative_pr opts..., ) if err != nil { + recordModelLoadFailure(appConfig, modelConfig.Name, modelConfig.Backend, err, nil) return nil, err } diff --git a/core/backend/llm.go b/core/backend/llm.go index 1316f72e4bf9..b82533a9fdc8 100644 --- a/core/backend/llm.go +++ b/core/backend/llm.go @@ -65,6 +65,7 @@ func ModelInference(ctx context.Context, s string, messages schema.Messages, ima opts := ModelOptions(*c, o) inferenceModel, err := loader.Load(opts...) if err != nil { + recordModelLoadFailure(o, c.Name, c.Backend, err, map[string]any{"model_file": modelFile}) return nil, err } diff --git a/core/backend/options.go b/core/backend/options.go index 85c67f60d0e8..4275a6f07495 100644 --- a/core/backend/options.go +++ b/core/backend/options.go @@ -1,17 +1,36 @@ package backend import ( - "strings" "math/rand" "os" "path/filepath" + "strings" + "time" "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/trace" pb "github.com/mudler/LocalAI/pkg/grpc/proto" "github.com/mudler/LocalAI/pkg/model" "github.com/mudler/xlog" ) +// recordModelLoadFailure records a backend trace when model loading fails. +func recordModelLoadFailure(appConfig *config.ApplicationConfig, modelName, backend string, err error, data map[string]any) { + if !appConfig.EnableTracing { + return + } + trace.InitBackendTracingIfEnabled(appConfig.TracingMaxItems) + trace.RecordBackendTrace(trace.BackendTrace{ + Timestamp: time.Now(), + Type: trace.BackendTraceModelLoad, + ModelName: modelName, + Backend: backend, + Summary: "Model load failed", + Error: err.Error(), + Data: data, + }) +} + func ModelOptions(c config.ModelConfig, so *config.ApplicationConfig, opts ...model.Option) []model.Option { name := c.Name if name == "" { diff --git a/core/backend/rerank.go b/core/backend/rerank.go index 2ff10120fdad..4b8f8b288b48 100644 --- a/core/backend/rerank.go +++ b/core/backend/rerank.go @@ -15,6 +15,7 @@ func Rerank(request *proto.RerankRequest, loader *model.ModelLoader, appConfig * opts := ModelOptions(modelConfig, appConfig) rerankModel, err := loader.Load(opts...) if err != nil { + recordModelLoadFailure(appConfig, modelConfig.Name, modelConfig.Backend, err, nil) return nil, err } diff --git a/core/backend/soundgeneration.go b/core/backend/soundgeneration.go index f2b03c45c5fd..f7f4d2f8228a 100644 --- a/core/backend/soundgeneration.go +++ b/core/backend/soundgeneration.go @@ -37,6 +37,7 @@ func SoundGeneration( opts := ModelOptions(modelConfig, appConfig) soundGenModel, err := loader.Load(opts...) if err != nil { + recordModelLoadFailure(appConfig, modelConfig.Name, modelConfig.Backend, err, nil) return "", nil, err } diff --git a/core/backend/token_metrics.go b/core/backend/token_metrics.go index c81f57cab50f..4a9289eecb03 100644 --- a/core/backend/token_metrics.go +++ b/core/backend/token_metrics.go @@ -18,6 +18,7 @@ func TokenMetrics( opts := ModelOptions(modelConfig, appConfig, model.WithModel(modelFile)) model, err := loader.Load(opts...) if err != nil { + recordModelLoadFailure(appConfig, modelConfig.Name, modelConfig.Backend, err, nil) return nil, err } diff --git a/core/backend/tokenize.go b/core/backend/tokenize.go index 60d8568f2ae8..f70ca14e2b6d 100644 --- a/core/backend/tokenize.go +++ b/core/backend/tokenize.go @@ -18,6 +18,7 @@ func ModelTokenize(s string, loader *model.ModelLoader, modelConfig config.Model opts := ModelOptions(modelConfig, appConfig) inferenceModel, err = loader.Load(opts...) if err != nil { + recordModelLoadFailure(appConfig, modelConfig.Name, modelConfig.Backend, err, nil) return schema.TokenizeResponse{}, err } diff --git a/core/backend/transcript.go b/core/backend/transcript.go index 7568e4e40706..6aa903e4a0de 100644 --- a/core/backend/transcript.go +++ b/core/backend/transcript.go @@ -23,6 +23,7 @@ func ModelTranscription(audio, language string, translate, diarize bool, prompt transcriptionModel, err := ml.Load(opts...) if err != nil { + recordModelLoadFailure(appConfig, modelConfig.Name, modelConfig.Backend, err, nil) return nil, err } diff --git a/core/backend/tts.go b/core/backend/tts.go index 69193db12a5d..2f3d31193b91 100644 --- a/core/backend/tts.go +++ b/core/backend/tts.go @@ -31,6 +31,7 @@ func ModelTTS( opts := ModelOptions(modelConfig, appConfig) ttsModel, err := loader.Load(opts...) if err != nil { + recordModelLoadFailure(appConfig, modelConfig.Name, modelConfig.Backend, err, nil) return "", nil, err } @@ -131,6 +132,7 @@ func ModelTTSStream( opts := ModelOptions(modelConfig, appConfig) ttsModel, err := loader.Load(opts...) if err != nil { + recordModelLoadFailure(appConfig, modelConfig.Name, modelConfig.Backend, err, nil) return err } diff --git a/core/backend/vad.go b/core/backend/vad.go index 37859931dc1b..bcf6f5976430 100644 --- a/core/backend/vad.go +++ b/core/backend/vad.go @@ -17,6 +17,7 @@ func VAD(request *schema.VADRequest, opts := ModelOptions(modelConfig, appConfig) vadModel, err := ml.Load(opts...) if err != nil { + recordModelLoadFailure(appConfig, modelConfig.Name, modelConfig.Backend, err, nil) return nil, err } diff --git a/core/backend/video.go b/core/backend/video.go index 277320515c2a..65677f0552d8 100644 --- a/core/backend/video.go +++ b/core/backend/video.go @@ -17,6 +17,7 @@ func VideoGeneration(height, width int32, prompt, negativePrompt, startImage, en opts..., ) if err != nil { + recordModelLoadFailure(appConfig, modelConfig.Name, modelConfig.Backend, err, nil) return nil, err } diff --git a/core/config/application_config.go b/core/config/application_config.go index 8edc22c00483..74c3511a6594 100644 --- a/core/config/application_config.go +++ b/core/config/application_config.go @@ -21,6 +21,7 @@ type ApplicationConfig struct { Debug bool EnableTracing bool TracingMaxItems int + EnableBackendLogging bool GeneratedContentDir string UploadDir string @@ -213,6 +214,10 @@ var EnableTracing = func(o *ApplicationConfig) { o.EnableTracing = true } +var EnableBackendLogging = func(o *ApplicationConfig) { + o.EnableBackendLogging = true +} + var EnableWatchDogIdleCheck = func(o *ApplicationConfig) { o.WatchDog = true o.WatchDogIdle = true @@ -743,6 +748,7 @@ func (o *ApplicationConfig) ToRuntimeSettings() RuntimeSettings { debug := o.Debug tracingMaxItems := o.TracingMaxItems enableTracing := o.EnableTracing + enableBackendLogging := o.EnableBackendLogging cors := o.CORS csrf := o.CSRF corsAllowOrigins := o.CORSAllowOrigins @@ -816,6 +822,7 @@ func (o *ApplicationConfig) ToRuntimeSettings() RuntimeSettings { Debug: &debug, TracingMaxItems: &tracingMaxItems, EnableTracing: &enableTracing, + EnableBackendLogging: &enableBackendLogging, CORS: &cors, CSRF: &csrf, CORSAllowOrigins: &corsAllowOrigins, @@ -944,6 +951,9 @@ func (o *ApplicationConfig) ApplyRuntimeSettings(settings *RuntimeSettings) (req if settings.TracingMaxItems != nil { o.TracingMaxItems = *settings.TracingMaxItems } + if settings.EnableBackendLogging != nil { + o.EnableBackendLogging = *settings.EnableBackendLogging + } if settings.CORS != nil { o.CORS = *settings.CORS } diff --git a/core/config/runtime_settings.go b/core/config/runtime_settings.go index ea1765719961..7637a0d94696 100644 --- a/core/config/runtime_settings.go +++ b/core/config/runtime_settings.go @@ -36,8 +36,9 @@ type RuntimeSettings struct { ContextSize *int `json:"context_size,omitempty"` F16 *bool `json:"f16,omitempty"` Debug *bool `json:"debug,omitempty"` - EnableTracing *bool `json:"enable_tracing,omitempty"` - TracingMaxItems *int `json:"tracing_max_items,omitempty"` + EnableTracing *bool `json:"enable_tracing,omitempty"` + TracingMaxItems *int `json:"tracing_max_items,omitempty"` + EnableBackendLogging *bool `json:"enable_backend_logging,omitempty"` // Security/CORS settings CORS *bool `json:"cors,omitempty"` diff --git a/core/http/endpoints/localai/settings.go b/core/http/endpoints/localai/settings.go index 0114b31ac485..2428c9cd4897 100644 --- a/core/http/endpoints/localai/settings.go +++ b/core/http/endpoints/localai/settings.go @@ -136,6 +136,12 @@ func UpdateSettingsEndpoint(app *application.Application) echo.HandlerFunc { appConfig.ApiKeys = append(envKeys, runtimeKeys...) } + // Update backend logging dynamically + if settings.EnableBackendLogging != nil { + app.ModelLoader().SetBackendLoggingEnabled(*settings.EnableBackendLogging) + xlog.Info("Updated backend logging setting", "enableBackendLogging", *settings.EnableBackendLogging) + } + // Update watchdog dynamically for settings that don't require restart if settings.ForceEvictionWhenBusy != nil { currentWD := app.ModelLoader().GetWatchDog() diff --git a/core/http/middleware/trace.go b/core/http/middleware/trace.go index 2d7bcf16719d..800b824c8789 100644 --- a/core/http/middleware/trace.go +++ b/core/http/middleware/trace.go @@ -29,8 +29,10 @@ type APIExchangeResponse struct { type APIExchange struct { Timestamp time.Time `json:"timestamp"` + Duration time.Duration `json:"duration"` Request APIExchangeRequest `json:"request"` Response APIExchangeResponse `json:"response"` + Error string `json:"error,omitempty"` } var traceBuffer *circularbuffer.Queue[APIExchange] @@ -108,13 +110,18 @@ func TraceMiddleware(app *application.Application) echo.MiddlewareFunc { } c.Response().Writer = mw - err = next(c) - if err != nil { - c.Response().Writer = mw.ResponseWriter // Restore original writer if error - return err + handlerErr := next(c) + + // Restore original writer unconditionally + c.Response().Writer = mw.ResponseWriter + + // Determine response status (use 500 if handler errored and no status was set) + status := c.Response().Status + if status == 0 && handlerErr != nil { + status = http.StatusInternalServerError } - // Create exchange log + // Create exchange log (always, even on error) requestHeaders := c.Request().Header.Clone() requestBody := make([]byte, len(body)) copy(requestBody, body) @@ -123,6 +130,7 @@ func TraceMiddleware(app *application.Application) echo.MiddlewareFunc { copy(responseBody, resBody.Bytes()) exchange := APIExchange{ Timestamp: startTime, + Duration: time.Since(startTime), Request: APIExchangeRequest{ Method: c.Request().Method, Path: c.Path(), @@ -130,11 +138,14 @@ func TraceMiddleware(app *application.Application) echo.MiddlewareFunc { Body: &requestBody, }, Response: APIExchangeResponse{ - Status: c.Response().Status, + Status: status, Headers: &responseHeaders, Body: &responseBody, }, } + if handlerErr != nil { + exchange.Error = handlerErr.Error() + } select { case logChan <- exchange: @@ -142,7 +153,7 @@ func TraceMiddleware(app *application.Application) echo.MiddlewareFunc { xlog.Warn("Trace channel full, dropping trace") } - return nil + return handlerErr } } } diff --git a/core/http/react-ui/e2e/backend-logs.spec.js b/core/http/react-ui/e2e/backend-logs.spec.js new file mode 100644 index 000000000000..3780cad550f7 --- /dev/null +++ b/core/http/react-ui/e2e/backend-logs.spec.js @@ -0,0 +1,64 @@ +import { test, expect } from '@playwright/test' + +test.describe('Backend Logs', () => { + test('model detail page shows title', async ({ page }) => { + await page.goto('/app/backend-logs/mock-model') + await expect(page.locator('.page-title')).toContainText('mock-model') + }) + + test('no back arrow link on detail page', async ({ page }) => { + await page.goto('/app/backend-logs/mock-model') + await expect(page.locator('a[href="/app/backend-logs"]')).not.toBeVisible() + }) + + test('filter buttons are visible', async ({ page }) => { + await page.goto('/app/backend-logs/mock-model') + await expect(page.locator('button', { hasText: 'All' })).toBeVisible() + await expect(page.locator('button', { hasText: 'stdout' })).toBeVisible() + await expect(page.locator('button', { hasText: 'stderr' })).toBeVisible() + }) + + test('filter buttons toggle active state', async ({ page }) => { + await page.goto('/app/backend-logs/mock-model') + + const allBtn = page.locator('button', { hasText: 'All' }) + const stdoutBtn = page.locator('button', { hasText: 'stdout' }) + + // All is active by default + await expect(allBtn).toHaveClass(/btn-primary/) + + // Click stdout + await stdoutBtn.click() + await expect(stdoutBtn).toHaveClass(/btn-primary/) + await expect(allBtn).not.toHaveClass(/btn-primary/) + }) + + test('export button is present', async ({ page }) => { + await page.goto('/app/backend-logs/mock-model') + await expect(page.locator('button', { hasText: 'Export' })).toBeVisible() + }) + + test('auto-scroll checkbox is present', async ({ page }) => { + await page.goto('/app/backend-logs/mock-model') + await expect(page.locator('text=Auto-scroll')).toBeVisible() + }) + + test('clear button is present', async ({ page }) => { + await page.goto('/app/backend-logs/mock-model') + await expect(page.locator('button', { hasText: 'Clear' })).toBeVisible() + }) + + test('details toggle button is present and toggles', async ({ page }) => { + await page.goto('/app/backend-logs/mock-model') + + // "Text only" button visible by default (details are shown) + const toggleBtn = page.locator('button', { hasText: 'Text only' }) + await expect(toggleBtn).toBeVisible() + + // Click to hide details + await toggleBtn.click() + + // Button label changes to "Show details" + await expect(page.locator('button', { hasText: 'Show details' })).toBeVisible() + }) +}) diff --git a/core/http/react-ui/e2e/manage-logs-link.spec.js b/core/http/react-ui/e2e/manage-logs-link.spec.js new file mode 100644 index 000000000000..d4cff23f9282 --- /dev/null +++ b/core/http/react-ui/e2e/manage-logs-link.spec.js @@ -0,0 +1,29 @@ +import { test, expect } from '@playwright/test' + +test.describe('Manage Page - Backend Logs Link', () => { + test('models table shows terminal icon for logs', async ({ page }) => { + await page.goto('/app/manage') + // Wait for models to load + await expect(page.locator('.table')).toBeVisible({ timeout: 10_000 }) + + // Check for terminal icon (backend logs link) + const terminalIcon = page.locator('a[title="Backend logs"] i.fa-terminal') + await expect(terminalIcon.first()).toBeVisible() + }) + + test('terminal icon links to backend-logs page', async ({ page }) => { + await page.goto('/app/manage') + await expect(page.locator('.table')).toBeVisible({ timeout: 10_000 }) + + const logsLink = page.locator('a[title="Backend logs"]').first() + await expect(logsLink).toBeVisible() + + // Link uses href="#" with onClick for navigation + const href = await logsLink.getAttribute('href') + expect(href).toBe('#') + + // Click and verify navigation + await logsLink.click() + await expect(page).toHaveURL(/\/app\/backend-logs\//) + }) +}) diff --git a/core/http/react-ui/e2e/settings-backend-logging.spec.js b/core/http/react-ui/e2e/settings-backend-logging.spec.js new file mode 100644 index 000000000000..b01b00f3908b --- /dev/null +++ b/core/http/react-ui/e2e/settings-backend-logging.spec.js @@ -0,0 +1,36 @@ +import { test, expect } from '@playwright/test' + +test.describe('Settings - Backend Logging', () => { + test.beforeEach(async ({ page }) => { + await page.goto('/app/settings') + // Wait for settings to load + await expect(page.locator('h3', { hasText: 'Tracing' })).toBeVisible({ timeout: 10_000 }) + }) + + test('backend logging toggle is visible in tracing section', async ({ page }) => { + await expect(page.locator('text=Enable Backend Logging')).toBeVisible() + }) + + test('backend logging toggle can be toggled', async ({ page }) => { + // Find the checkbox associated with backend logging + const section = page.locator('div', { has: page.locator('text=Enable Backend Logging') }) + const checkbox = section.locator('input[type="checkbox"]').last() + + // Toggle on + const wasChecked = await checkbox.isChecked() + await checkbox.locator('..').click() + if (wasChecked) { + await expect(checkbox).not.toBeChecked() + } else { + await expect(checkbox).toBeChecked() + } + }) + + test('save shows toast', async ({ page }) => { + // Click save button + await page.locator('button', { hasText: 'Save' }).click() + + // Verify toast appears + await expect(page.locator('text=Settings saved')).toBeVisible({ timeout: 5_000 }) + }) +}) diff --git a/core/http/react-ui/e2e/traces-errors.spec.js b/core/http/react-ui/e2e/traces-errors.spec.js new file mode 100644 index 000000000000..af820663536c --- /dev/null +++ b/core/http/react-ui/e2e/traces-errors.spec.js @@ -0,0 +1,50 @@ +import { test, expect } from '@playwright/test' + +test.describe('Traces - Error Display', () => { + test.beforeEach(async ({ page }) => { + // Mock API traces with sample data so the table renders + await page.route('**/api/traces', (route) => { + route.fulfill({ + contentType: 'application/json', + body: JSON.stringify([ + { + request: { method: 'POST', path: '/v1/chat/completions' }, + response: { status: 200 }, + error: null, + }, + ]), + }) + }) + // Mock backend traces with sample data + await page.route('**/api/backend-traces', (route) => { + route.fulfill({ + contentType: 'application/json', + body: JSON.stringify([ + { + type: 'model_load', + timestamp: Date.now() * 1_000_000, + model_name: 'mock-model', + summary: 'Loaded model', + duration: 500_000_000, + error: null, + }, + ]), + }) + }) + await page.goto('/app/traces') + await expect(page.locator('text=Tracing is')).toBeVisible({ timeout: 10_000 }) + }) + + test('API traces tab has Result column header', async ({ page }) => { + // API tab is active by default + await expect(page.locator('th', { hasText: 'Result' })).toBeVisible() + }) + + test('backend traces tab shows model_load type if present', async ({ page }) => { + // Switch to backend traces tab + await page.locator('button', { hasText: 'Backend Traces' }).click() + + // The table should be visible with Type column + await expect(page.locator('th', { hasText: 'Type' })).toBeVisible() + }) +}) diff --git a/core/http/react-ui/src/App.css b/core/http/react-ui/src/App.css index 2ff91d98bc08..375575c107ec 100644 --- a/core/http/react-ui/src/App.css +++ b/core/http/react-ui/src/App.css @@ -2109,7 +2109,7 @@ height: 4px; appearance: none; -webkit-appearance: none; - background: var(--color-bg-tertiary); + background: var(--color-border-default); border-radius: 2px; outline: none; } diff --git a/core/http/react-ui/src/pages/BackendLogs.jsx b/core/http/react-ui/src/pages/BackendLogs.jsx new file mode 100644 index 000000000000..ab38d8cd8afe --- /dev/null +++ b/core/http/react-ui/src/pages/BackendLogs.jsx @@ -0,0 +1,297 @@ +import { useState, useEffect, useCallback, useRef, useMemo } from 'react' +import { useParams, useSearchParams, useOutletContext, Link } from 'react-router-dom' +import { backendLogsApi } from '../utils/api' +import { formatTimestamp } from '../utils/format' +import { apiUrl } from '../utils/basePath' +import LoadingSpinner from '../components/LoadingSpinner' + +function wsUrl(path) { + const proto = window.location.protocol === 'https:' ? 'wss:' : 'ws:' + return `${proto}//${window.location.host}${apiUrl(path)}` +} + +const STREAM_BADGE = { + stdout: { bg: 'rgba(59,130,246,0.15)', color: '#60a5fa', label: 'stdout' }, + stderr: { bg: 'rgba(239,68,68,0.15)', color: '#f87171', label: 'stderr' }, +} + +// Detail view: log lines for a specific model +function BackendLogsDetail({ modelId }) { + const { addToast } = useOutletContext() + const [searchParams] = useSearchParams() + const fromTimestamp = searchParams.get('from') + + const [lines, setLines] = useState([]) + const [loading, setLoading] = useState(true) + const [filter, setFilter] = useState('all') + const [autoScroll, setAutoScroll] = useState(true) + const [showDetails, setShowDetails] = useState(true) + const [wsConnected, setWsConnected] = useState(false) + const logContainerRef = useRef(null) + const wsRef = useRef(null) + const reconnectTimerRef = useRef(null) + const loadingRef = useRef(true) + const scrolledToTimestampRef = useRef(false) + const pendingLinesRef = useRef([]) + const flushTimerRef = useRef(null) + + // Keep loadingRef in sync + useEffect(() => { loadingRef.current = loading }, [loading]) + + // Auto-scroll to bottom when new lines arrive + useEffect(() => { + if (autoScroll && logContainerRef.current) { + logContainerRef.current.scrollTop = logContainerRef.current.scrollHeight + } + }, [lines, autoScroll]) + + // WebSocket connection with reconnect + const connectWebSocket = useCallback(() => { + if (wsRef.current && wsRef.current.readyState <= 1) return + + const url = wsUrl(`/ws/backend-logs/${encodeURIComponent(modelId)}`) + const ws = new WebSocket(url) + wsRef.current = ws + + ws.onopen = () => { + setWsConnected(true) + setLoading(false) + } + + ws.onmessage = (event) => { + try { + const msg = JSON.parse(event.data) + if (msg.type === 'initial') { + setLines(Array.isArray(msg.lines) ? msg.lines : []) + setLoading(false) + } else if (msg.type === 'line' && msg.line) { + // Batch incoming lines to reduce renders + pendingLinesRef.current.push(msg.line) + if (!flushTimerRef.current) { + flushTimerRef.current = requestAnimationFrame(() => { + const batch = pendingLinesRef.current + pendingLinesRef.current = [] + flushTimerRef.current = null + setLines(prev => prev.concat(batch)) + }) + } + } + } catch { + // ignore parse errors + } + } + + ws.onclose = () => { + setWsConnected(false) + reconnectTimerRef.current = setTimeout(connectWebSocket, 3000) + } + + ws.onerror = () => { + // Fall back to REST if WebSocket fails on first connect + if (loadingRef.current) { + backendLogsApi.getLines(modelId) + .then(data => setLines(Array.isArray(data) ? data : [])) + .catch(() => {}) + .finally(() => setLoading(false)) + } + } + }, [modelId]) + + useEffect(() => { + connectWebSocket() + return () => { + if (wsRef.current) wsRef.current.close() + if (reconnectTimerRef.current) clearTimeout(reconnectTimerRef.current) + if (flushTimerRef.current) cancelAnimationFrame(flushTimerRef.current) + } + }, [connectWebSocket]) + + // Scroll to timestamp if `from` query param is set (once) + useEffect(() => { + if (!fromTimestamp || scrolledToTimestampRef.current || !logContainerRef.current || lines.length === 0) return + const fromDate = new Date(fromTimestamp).getTime() + const lineElements = logContainerRef.current.querySelectorAll('[data-log-line]') + for (const el of lineElements) { + const lineTime = new Date(el.dataset.timestamp).getTime() + if (lineTime >= fromDate) { + el.scrollIntoView({ behavior: 'smooth', block: 'start' }) + el.style.background = 'rgba(59,130,246,0.1)' + setTimeout(() => { el.style.background = '' }, 3000) + scrolledToTimestampRef.current = true + break + } + } + }, [fromTimestamp, lines]) + + const filteredLines = useMemo( + () => filter === 'all' ? lines : lines.filter(l => l.stream === filter), + [lines, filter] + ) + + const handleClear = async () => { + try { + await backendLogsApi.clear(modelId) + setLines([]) + addToast('Logs cleared', 'success') + } catch (err) { + addToast(`Failed to clear: ${err.message}`, 'error') + } + } + + const handleExport = () => { + const blob = new Blob([JSON.stringify(filteredLines, null, 2)], { type: 'application/json' }) + const url = URL.createObjectURL(blob) + const a = document.createElement('a') + a.href = url + a.download = `backend-logs-${modelId}-${new Date().toISOString().slice(0, 10)}.json` + a.click() + URL.revokeObjectURL(url) + } + + return ( +
+
+
+

+ + {modelId} +

+

Backend process output

+
+
+ + {/* Toolbar */} +
+
+ {['all', 'stdout', 'stderr'].map(f => ( + + ))} +
+ + + +
+ + + {wsConnected ? 'Live' : 'Reconnecting...'} + + +
+
+ + {/* Log output */} + {loading ? ( +
+ +
+ ) : filteredLines.length === 0 ? ( +
+
+

No log lines

+

+ {filter !== 'all' + ? `No ${filter} output. Try switching to "All".` + : 'Log output will appear here as the backend process runs.'} +

+
+ ) : ( +
+ {filteredLines.map((line, i) => { + const badge = STREAM_BADGE[line.stream] || STREAM_BADGE.stdout + return ( +
+ {showDetails && (<> + + {formatTimestamp(line.timestamp)} + + + {badge.label} + + )} + + {line.text} + +
+ ) + })} +
+ )} +
+ ) +} + +export default function BackendLogs() { + const { modelId } = useParams() + + if (modelId) { + return + } + + // No model specified — redirect to System page + return ( +
+
+
+

No model selected

+

+ View backend logs for a specific model from the{' '} + System page. +

+
+
+ ) +} diff --git a/core/http/react-ui/src/pages/Backends.jsx b/core/http/react-ui/src/pages/Backends.jsx index 6b8f3e18e2fd..73d6e809720c 100644 --- a/core/http/react-ui/src/pages/Backends.jsx +++ b/core/http/react-ui/src/pages/Backends.jsx @@ -1,10 +1,10 @@ import { useState, useEffect, useCallback, useRef } from 'react' import { useNavigate, useOutletContext } from 'react-router-dom' import { backendsApi } from '../utils/api' +import React from 'react' import { useOperations } from '../hooks/useOperations' import LoadingSpinner from '../components/LoadingSpinner' import { renderMarkdown } from '../utils/markdown' -import Modal from '../components/Modal' export default function Backends() { const { addToast } = useOutletContext() @@ -21,7 +21,7 @@ export default function Backends() { const [manualUri, setManualUri] = useState('') const [manualName, setManualName] = useState('') const [manualAlias, setManualAlias] = useState('') - const [selectedBackend, setSelectedBackend] = useState(null) + const [expandedRow, setExpandedRow] = useState(null) const debounceRef = useRef(null) const [allBackends, setAllBackends] = useState([]) @@ -246,6 +246,7 @@ export default function Backends() { + Backend @@ -256,12 +257,21 @@ export default function Backends() { - {backends.map(b => { + {backends.map((b, idx) => { const op = getBackendOp(b) const isProcessing = !!op + const isExpanded = expandedRow === idx return ( - + + setExpandedRow(isExpanded ? null : idx)} + style={{ cursor: 'pointer' }} + > + {/* Chevron */} + {/* Icon */} {/* Description */} @@ -343,10 +348,7 @@ export default function Backends() { {/* Actions */} + {/* Expanded detail row */} + {isExpanded && ( + + + + )} + ) })} @@ -389,106 +400,67 @@ export default function Backends() { )} - {/* Detail Modal */} - {selectedBackend && ( - setSelectedBackend(null)}> -
-
-
- {selectedBackend.icon ? ( - - ) : ( -
- -
- )} -
-

{selectedBackend.name || selectedBackend.id}

- {selectedBackend.installed && Installed} -
-
- -
+
+ ) +} - {/* Description */} - {selectedBackend.description && ( -
-
-
- )} +function BackendDetailRow({ label, children }) { + if (!children) return null + return ( +
+ + + + ) +} - {/* Tags */} - {selectedBackend.tags && selectedBackend.tags.length > 0 && ( -
- Tags -
- {selectedBackend.tags.map(tag => ( - {tag} - ))} -
+function BackendDetail({ backend }) { + return ( +
+
Description
+ + {b.icon ? ( @@ -279,12 +289,7 @@ export default function Backends() { {/* Name */} - setSelectedBackend(b)} - > - {b.name || b.id} - + {b.name || b.id} -
- +
e.stopPropagation()}> {b.installed ? ( <>
+ +
+ {label} + {children}
+ + + {backend.description && ( +
+ )} + + + {backend.gallery && ( + + {typeof backend.gallery === 'string' ? backend.gallery : backend.gallery.name || '-'} + + )} + + + {backend.license && {backend.license}} + + + {backend.tags?.length > 0 && ( +
+ {backend.tags.map(tag => ( + {tag} + ))}
)} - - {/* URLs */} - {selectedBackend.urls && selectedBackend.urls.length > 0 && ( -
- Links -
- {selectedBackend.urls.map((url, i) => ( - - {url} - - ))} -
+ + + {backend.urls?.length > 0 && ( +
+ {backend.urls.map((url, i) => ( + + {url} + + ))}
)} - - {/* Repository / License */} -
- {selectedBackend.gallery && ( -
- Repository -

{typeof selectedBackend.gallery === 'string' ? selectedBackend.gallery : selectedBackend.gallery.name || '-'}

-
- )} - {selectedBackend.license && ( -
- License -

{selectedBackend.license}

-
- )} -
- - {/* Actions */} -
- {selectedBackend.installed ? ( - <> - - - - ) : ( - - )} - -
-
- - )} + +
+
) } diff --git a/core/http/react-ui/src/pages/Chat.jsx b/core/http/react-ui/src/pages/Chat.jsx index 006dff0ffe72..071d6830a6ee 100644 --- a/core/http/react-ui/src/pages/Chat.jsx +++ b/core/http/react-ui/src/pages/Chat.jsx @@ -354,6 +354,9 @@ export default function Chat() { modelsApi.getConfigJson(model).then(cfg => { if (cancelled) return setModelInfo(cfg) + if (cfg?.context_size > 0 && activeChat) { + updateChatSettings(activeChat.id, { contextSize: cfg.context_size }) + } const hasMcp = !!(cfg?.mcp?.remote || cfg?.mcp?.stdio) setMcpAvailable(hasMcp) if (!hasMcp && activeChat?.mcpMode) { diff --git a/core/http/react-ui/src/pages/Manage.jsx b/core/http/react-ui/src/pages/Manage.jsx index 00b62c9cdf30..f062b7640ffb 100644 --- a/core/http/react-ui/src/pages/Manage.jsx +++ b/core/http/react-ui/src/pages/Manage.jsx @@ -204,6 +204,14 @@ export default function Manage() { > + { e.preventDefault(); navigate(`/app/backend-logs/${encodeURIComponent(model.id)}`) }} + style={{ fontSize: '0.75rem', color: 'var(--color-primary)' }} + title="Backend logs" + > + + diff --git a/core/http/react-ui/src/pages/Models.jsx b/core/http/react-ui/src/pages/Models.jsx index eec4ec43122f..9ee6d9800e85 100644 --- a/core/http/react-ui/src/pages/Models.jsx +++ b/core/http/react-ui/src/pages/Models.jsx @@ -3,8 +3,7 @@ import { useNavigate, useOutletContext } from 'react-router-dom' import { modelsApi } from '../utils/api' import { useOperations } from '../hooks/useOperations' import { useResources } from '../hooks/useResources' -import { formatBytes } from '../utils/format' -import Modal from '../components/Modal' +import React from 'react' const LOADING_PHRASES = [ @@ -135,7 +134,8 @@ export default function Models() { const [sort, setSort] = useState('') const [order, setOrder] = useState('asc') const [installing, setInstalling] = useState(new Set()) - const [selectedModel, setSelectedModel] = useState(null) + const [expandedRow, setExpandedRow] = useState(null) + const [expandedFiles, setExpandedFiles] = useState(false) const [stats, setStats] = useState({ total: 0, installed: 0, repositories: 0 }) const debounceRef = useRef(null) @@ -322,6 +322,7 @@ export default function Models() { + - {models.map(model => { + {models.map((model, idx) => { const name = model.name || model.id const installing = isInstalling(name) const progress = getOperationProgress(name) const fit = fitsGpu(model.estimated_vram_bytes) + const isExpanded = expandedRow === idx return ( - + + { setExpandedRow(isExpanded ? null : idx); setExpandedFiles(false) }} + style={{ cursor: 'pointer' }} + > + {/* Chevron */} + {/* Icon */} + {/* Expanded detail row */} + {isExpanded && ( + + + + )} + ) })} @@ -493,74 +505,125 @@ export default function Models() { )} - {/* Detail Modal */} - {selectedModel && ( - setSelectedModel(null)}> - {/* Modal header */} -
-

{selectedModel.name}

- -
- {/* Modal body */} -
- {/* Icon */} - {selectedModel.icon && ( -
- -
- )} - {/* Description */} - {selectedModel.description && ( -

- {selectedModel.description} -

+
+ ) +} + +function DetailRow({ label, children }) { + if (!children) return null + return ( + + + + + ) +} + +function ModelDetail({ model, fit, expandedFiles, setExpandedFiles }) { + const files = model.additionalFiles || model.files || [] + return ( +
+
handleSort('name')}> Model Name {sort === 'name' && } @@ -335,14 +336,23 @@ export default function Models() {
+ +
-
- +
e.stopPropagation()}> {model.installed ? ( <>
+ +
+ {label} + {children}
+ + + {model.description && ( + {model.description} )} - {/* Size/VRAM */} - {(selectedModel.estimated_size_display || selectedModel.estimated_vram_display) && ( -
- {selectedModel.estimated_size_display &&
Size: {selectedModel.estimated_size_display}
} - {selectedModel.estimated_vram_display &&
VRAM: {selectedModel.estimated_vram_display}
} -
+ + + {model.gallery && ( + + {typeof model.gallery === 'string' ? model.gallery : model.gallery.name || '—'} + )} - {/* Tags */} - {selectedModel.tags?.length > 0 && ( -
- {selectedModel.tags.map(tag => ( - {tag} + + + {model.estimated_size_display && model.estimated_size_display !== '0 B' ? model.estimated_size_display : null} + + + {model.estimated_vram_display && model.estimated_vram_display !== '0 B' ? ( + + {model.estimated_vram_display} + {fit !== null && ( + + {fit ? 'Fits in GPU' : 'May not fit in GPU'} + + )} + + ) : null} + + + {model.license && {model.license}} + + + {model.tags?.length > 0 && ( +
+ {model.tags.map(tag => ( + {tag} ))}
)} - {/* Links */} - {selectedModel.urls?.length > 0 && ( - - {/* Modal footer */} -
- -
- - )} + + {model.trustRemoteCode && ( + + + Requires Trust Remote Code + + + )} + {files.length > 0 && ( + +
+ + {expandedFiles && ( +
+
+ + + + + + + + + {files.map((f, i) => ( + + + + + + ))} + +
FilenameURISHA256
{f.filename || '—'}{f.uri || '—'} + {f.sha256 ? f.sha256.substring(0, 16) + '...' : '—'} +
+ + )} + + + )} + + ) } diff --git a/core/http/react-ui/src/pages/Settings.jsx b/core/http/react-ui/src/pages/Settings.jsx index 3de33ea553ec..b1491f216317 100644 --- a/core/http/react-ui/src/pages/Settings.jsx +++ b/core/http/react-ui/src/pages/Settings.jsx @@ -300,6 +300,9 @@ export default function Settings() { update('tracing_max_items', parseInt(e.target.value) || 0)} placeholder="100" disabled={!settings.enable_tracing} /> + + update('enable_backend_logging', v)} /> + diff --git a/core/http/react-ui/src/pages/Traces.jsx b/core/http/react-ui/src/pages/Traces.jsx index 3277eed32721..a18df71b2de3 100644 --- a/core/http/react-ui/src/pages/Traces.jsx +++ b/core/http/react-ui/src/pages/Traces.jsx @@ -1,6 +1,7 @@ import React, { useState, useEffect, useCallback, useRef } from 'react' import { useOutletContext } from 'react-router-dom' import { tracesApi, settingsApi } from '../utils/api' +import { formatTimestamp } from '../utils/format' import LoadingSpinner from '../components/LoadingSpinner' import Toggle from '../components/Toggle' import SettingRow from '../components/SettingRow' @@ -19,12 +20,6 @@ function formatDuration(ns) { return `${(ns / 1_000_000_000).toFixed(2)}s` } -function formatTimestamp(ts) { - if (!ts) return '-' - const d = new Date(ts) - return d.toLocaleTimeString() + '.' + String(d.getMilliseconds()).padStart(3, '0') -} - function decodeTraceBody(body) { if (!body) return '' try { @@ -75,6 +70,8 @@ const TYPE_COLORS = { sound_generation: { bg: 'rgba(20,184,166,0.15)', color: '#2dd4bf' }, rerank: { bg: 'rgba(99,102,241,0.15)', color: '#818cf8' }, tokenize: { bg: 'rgba(107,114,128,0.15)', color: '#9ca3af' }, + detection: { bg: 'rgba(14,165,233,0.15)', color: '#38bdf8' }, + model_load: { bg: 'rgba(239,68,68,0.15)', color: '#f87171' }, } function typeBadgeStyle(type) { @@ -221,6 +218,18 @@ function BackendTraceDetail({ trace }) { )} + {/* Backend logs link */} + {trace.model_name && ( + + )} + {/* Audio snippet */} {trace.data && } @@ -234,6 +243,16 @@ function BackendTraceDetail({ trace }) { function ApiTraceDetail({ trace }) { return (
+ {trace.error && ( +
+ + {trace.error} +
+ )}

Request Body

@@ -452,6 +471,7 @@ export default function Traces() { Method Path Status + Result @@ -462,10 +482,15 @@ export default function Traces() { {trace.request?.method || '-'} {trace.request?.path || '-'} {trace.response?.status || '-'} + + {trace.error + ? + : } + {expandedRow === i && ( - + diff --git a/core/http/react-ui/src/router.jsx b/core/http/react-ui/src/router.jsx index 4639f914d5fa..ff2d763037b6 100644 --- a/core/http/react-ui/src/router.jsx +++ b/core/http/react-ui/src/router.jsx @@ -27,6 +27,7 @@ import AgentTaskDetails from './pages/AgentTaskDetails' import AgentJobDetails from './pages/AgentJobDetails' import ModelEditor from './pages/ModelEditor' import ImportModel from './pages/ImportModel' +import BackendLogs from './pages/BackendLogs' import Explorer from './pages/Explorer' import Login from './pages/Login' import NotFound from './pages/NotFound' @@ -54,6 +55,7 @@ const appChildren = [ { path: 'backends', element: }, { path: 'settings', element: }, { path: 'traces', element: }, + { path: 'backend-logs/:modelId', element: }, { path: 'p2p', element: }, { path: 'agents', element: }, { path: 'agents/new', element: }, diff --git a/core/http/react-ui/src/utils/api.js b/core/http/react-ui/src/utils/api.js index 2c949611b418..3680087b42df 100644 --- a/core/http/react-ui/src/utils/api.js +++ b/core/http/react-ui/src/utils/api.js @@ -140,6 +140,13 @@ export const settingsApi = { save: (body) => postJSON(API_CONFIG.endpoints.settings, body), } +// Backend Logs API +export const backendLogsApi = { + listModels: () => fetchJSON(API_CONFIG.endpoints.backendLogs), + getLines: (modelId) => fetchJSON(API_CONFIG.endpoints.backendLogsModel(modelId)), + clear: (modelId) => postJSON(API_CONFIG.endpoints.clearBackendLogs(modelId), {}), +} + // Traces API export const tracesApi = { get: () => fetchJSON(API_CONFIG.endpoints.traces), diff --git a/core/http/react-ui/src/utils/config.js b/core/http/react-ui/src/utils/config.js index 37b57bd0d63b..c9462ccfa1db 100644 --- a/core/http/react-ui/src/utils/config.js +++ b/core/http/react-ui/src/utils/config.js @@ -33,6 +33,11 @@ export const API_CONFIG = { backendTraces: '/api/backend-traces', clearBackendTraces: '/api/backend-traces/clear', + // Backend Logs + backendLogs: '/api/backend-logs', + backendLogsModel: (modelId) => `/api/backend-logs/${encodeURIComponent(modelId)}`, + clearBackendLogs: (modelId) => `/api/backend-logs/${encodeURIComponent(modelId)}/clear`, + // P2P p2pWorkers: '/api/p2p/workers', p2pFederation: '/api/p2p/federation', diff --git a/core/http/react-ui/src/utils/format.js b/core/http/react-ui/src/utils/format.js index 3372f86c6158..1ad8dac54d52 100644 --- a/core/http/react-ui/src/utils/format.js +++ b/core/http/react-ui/src/utils/format.js @@ -12,6 +12,12 @@ export function percentColor(pct) { return 'var(--color-success)' } +export function formatTimestamp(ts) { + if (!ts) return '-' + const d = new Date(ts) + return d.toLocaleTimeString() + '.' + String(d.getMilliseconds()).padStart(3, '0') +} + export function vendorColor(vendor) { if (!vendor) return 'var(--color-accent)' const v = vendor.toLowerCase() diff --git a/core/http/routes/ui.go b/core/http/routes/ui.go index 274fcaa0c598..ec6554665a49 100644 --- a/core/http/routes/ui.go +++ b/core/http/routes/ui.go @@ -2,16 +2,29 @@ package routes import ( "cmp" + "encoding/json" + "fmt" + "net/http" "slices" + "sync" + "time" + "github.com/gorilla/websocket" "github.com/labstack/echo/v4" "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/http/middleware" "github.com/mudler/LocalAI/core/services" "github.com/mudler/LocalAI/core/trace" "github.com/mudler/LocalAI/pkg/model" + "github.com/mudler/xlog" ) +var backendLogsUpgrader = websocket.Upgrader{ + CheckOrigin: func(r *http.Request) bool { + return true + }, +} + func RegisterUIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model.ModelLoader, @@ -74,4 +87,119 @@ func RegisterUIRoutes(app *echo.Echo, return c.NoContent(204) }) + // Backend logs REST endpoints + app.GET("/api/backend-logs", func(c echo.Context) error { + return c.JSON(200, ml.BackendLogs().ListModels()) + }) + + app.GET("/api/backend-logs/:modelId", func(c echo.Context) error { + modelID := c.Param("modelId") + return c.JSON(200, ml.BackendLogs().GetLines(modelID)) + }) + + app.POST("/api/backend-logs/:modelId/clear", func(c echo.Context) error { + ml.BackendLogs().Clear(c.Param("modelId")) + return c.NoContent(204) + }) + + // Backend logs WebSocket endpoint for real-time streaming + app.GET("/ws/backend-logs/:modelId", func(c echo.Context) error { + modelID := c.Param("modelId") + + ws, err := backendLogsUpgrader.Upgrade(c.Response(), c.Request(), nil) + if err != nil { + return err + } + defer ws.Close() + + ws.SetReadLimit(4096) + + // Set up ping/pong for keepalive + ws.SetReadDeadline(time.Now().Add(90 * time.Second)) + ws.SetPongHandler(func(string) error { + ws.SetReadDeadline(time.Now().Add(90 * time.Second)) + return nil + }) + + conn := &backendLogsConn{Conn: ws} + + // Send existing lines as initial batch + existingLines := ml.BackendLogs().GetLines(modelID) + initialMsg := map[string]any{ + "type": "initial", + "lines": existingLines, + } + if err := conn.writeJSON(initialMsg); err != nil { + xlog.Debug("WebSocket backend-logs initial write failed", "error", err) + return nil + } + + // Subscribe to new lines + lineCh, unsubscribe := ml.BackendLogs().Subscribe(modelID) + defer unsubscribe() + + // Handle close from client side + closeCh := make(chan struct{}) + go func() { + for { + _, _, err := ws.ReadMessage() + if err != nil { + close(closeCh) + return + } + } + }() + + // Ping ticker for keepalive + pingTicker := time.NewTicker(30 * time.Second) + defer pingTicker.Stop() + + // Forward new lines to WebSocket + for { + select { + case line, ok := <-lineCh: + if !ok { + return nil + } + lineMsg := map[string]any{ + "type": "line", + "line": line, + } + if err := conn.writeJSON(lineMsg); err != nil { + xlog.Debug("WebSocket backend-logs write error", "error", err) + return nil + } + case <-pingTicker.C: + if err := conn.writePing(); err != nil { + return nil + } + case <-closeCh: + return nil + } + } + }) +} + +// backendLogsConn wraps a websocket connection with a mutex for safe concurrent writes +type backendLogsConn struct { + *websocket.Conn + mu sync.Mutex +} + +func (c *backendLogsConn) writeJSON(v any) error { + c.mu.Lock() + defer c.mu.Unlock() + c.Conn.SetWriteDeadline(time.Now().Add(30 * time.Second)) + data, err := json.Marshal(v) + if err != nil { + return fmt.Errorf("marshal error: %w", err) + } + return c.Conn.WriteMessage(websocket.TextMessage, data) +} + +func (c *backendLogsConn) writePing() error { + c.mu.Lock() + defer c.mu.Unlock() + c.Conn.SetWriteDeadline(time.Now().Add(30 * time.Second)) + return c.Conn.WriteMessage(websocket.PingMessage, nil) } diff --git a/core/trace/backend_trace.go b/core/trace/backend_trace.go index 4e6237f9f700..33bbe5b176f2 100644 --- a/core/trace/backend_trace.go +++ b/core/trace/backend_trace.go @@ -23,6 +23,8 @@ const ( BackendTraceSoundGeneration BackendTraceType = "sound_generation" BackendTraceRerank BackendTraceType = "rerank" BackendTraceTokenize BackendTraceType = "tokenize" + BackendTraceDetection BackendTraceType = "detection" + BackendTraceModelLoad BackendTraceType = "model_load" ) type BackendTrace struct { diff --git a/pkg/model/backend_log_store.go b/pkg/model/backend_log_store.go new file mode 100644 index 000000000000..c20b387d8c20 --- /dev/null +++ b/pkg/model/backend_log_store.go @@ -0,0 +1,168 @@ +package model + +import ( + "sort" + "sync" + "time" + + "github.com/emirpasic/gods/v2/queues/circularbuffer" +) + +// BackendLogLine represents a single line of output from a backend process. +type BackendLogLine struct { + Timestamp time.Time `json:"timestamp"` + Stream string `json:"stream"` // "stdout" or "stderr" + Text string `json:"text"` +} + +// backendLogBuffer wraps a circular buffer for a single model's logs +// and tracks subscribers for real-time streaming. +type backendLogBuffer struct { + mu sync.Mutex + queue *circularbuffer.Queue[BackendLogLine] + subscribers map[int]chan BackendLogLine + nextSubID int +} + +// BackendLogStore stores per-model backend process output in circular buffers +// and supports real-time subscriptions for WebSocket streaming. +type BackendLogStore struct { + mu sync.RWMutex // protects the buffers map only + buffers map[string]*backendLogBuffer + maxLines int +} + +// NewBackendLogStore creates a new BackendLogStore with a maximum number of +// lines retained per model. +func NewBackendLogStore(maxLinesPerModel int) *BackendLogStore { + if maxLinesPerModel <= 0 { + maxLinesPerModel = 1000 + } + return &BackendLogStore{ + buffers: make(map[string]*backendLogBuffer), + maxLines: maxLinesPerModel, + } +} + +// getOrCreateBuffer returns the buffer for modelID, creating it if needed. +func (s *BackendLogStore) getOrCreateBuffer(modelID string) *backendLogBuffer { + s.mu.RLock() + buf, ok := s.buffers[modelID] + s.mu.RUnlock() + if ok { + return buf + } + + s.mu.Lock() + buf, ok = s.buffers[modelID] + if !ok { + buf = &backendLogBuffer{ + queue: circularbuffer.New[BackendLogLine](s.maxLines), + subscribers: make(map[int]chan BackendLogLine), + } + s.buffers[modelID] = buf + } + s.mu.Unlock() + return buf +} + +// AppendLine adds a log line for the given model. The buffer is lazily created. +// All active subscribers for this model are notified (non-blocking). +func (s *BackendLogStore) AppendLine(modelID, stream, text string) { + line := BackendLogLine{ + Timestamp: time.Now(), + Stream: stream, + Text: text, + } + + buf := s.getOrCreateBuffer(modelID) + buf.mu.Lock() + buf.queue.Enqueue(line) + for _, ch := range buf.subscribers { + select { + case ch <- line: + default: + } + } + buf.mu.Unlock() +} + +// GetLines returns a copy of all log lines for a model, or an empty slice. +func (s *BackendLogStore) GetLines(modelID string) []BackendLogLine { + s.mu.RLock() + buf, ok := s.buffers[modelID] + s.mu.RUnlock() + if !ok { + return []BackendLogLine{} + } + + buf.mu.Lock() + lines := buf.queue.Values() + buf.mu.Unlock() + return lines +} + +// ListModels returns a sorted list of model IDs that have log buffers. +func (s *BackendLogStore) ListModels() []string { + s.mu.RLock() + models := make([]string, 0, len(s.buffers)) + for id := range s.buffers { + models = append(models, id) + } + s.mu.RUnlock() + + sort.Strings(models) + return models +} + +// Clear removes all log lines for a model but keeps the buffer entry. +func (s *BackendLogStore) Clear(modelID string) { + s.mu.RLock() + buf, ok := s.buffers[modelID] + s.mu.RUnlock() + if !ok { + return + } + buf.mu.Lock() + buf.queue.Clear() + buf.mu.Unlock() +} + +// Remove deletes the buffer entry for a model entirely. +func (s *BackendLogStore) Remove(modelID string) { + s.mu.Lock() + if buf, ok := s.buffers[modelID]; ok { + buf.mu.Lock() + for _, ch := range buf.subscribers { + close(ch) + } + buf.mu.Unlock() + delete(s.buffers, modelID) + } + s.mu.Unlock() +} + +// Subscribe returns a channel that receives new log lines for the given model +// in real-time, plus an unsubscribe function. The channel has a buffer of 100 +// lines to absorb short bursts without blocking the writer. +func (s *BackendLogStore) Subscribe(modelID string) (chan BackendLogLine, func()) { + ch := make(chan BackendLogLine, 100) + + buf := s.getOrCreateBuffer(modelID) + buf.mu.Lock() + id := buf.nextSubID + buf.nextSubID++ + buf.subscribers[id] = ch + buf.mu.Unlock() + + unsubscribe := func() { + buf.mu.Lock() + if _, exists := buf.subscribers[id]; exists { + delete(buf.subscribers, id) + close(ch) + } + buf.mu.Unlock() + } + + return ch, unsubscribe +} diff --git a/pkg/model/loader.go b/pkg/model/loader.go index 4a443ef73f83..79578965ea71 100644 --- a/pkg/model/loader.go +++ b/pkg/model/loader.go @@ -8,6 +8,7 @@ import ( "path/filepath" "strings" "sync" + "sync/atomic" "time" "github.com/mudler/LocalAI/pkg/system" @@ -33,6 +34,8 @@ type ModelLoader struct { lruEvictionMaxRetries int // Maximum number of retries when waiting for busy models lruEvictionRetryInterval time.Duration // Interval between retries when waiting for busy models onUnloadHooks []ModelUnloadHook + backendLogs *BackendLogStore + backendLoggingEnabled atomic.Bool } // NewModelLoader creates a new ModelLoader instance. @@ -45,6 +48,7 @@ func NewModelLoader(system *system.SystemState) *ModelLoader { externalBackends: make(map[string]string), lruEvictionMaxRetries: 30, // Default: 30 retries lruEvictionRetryInterval: 1 * time.Second, // Default: 1 second + backendLogs: NewBackendLogStore(1000), } return nml @@ -72,6 +76,18 @@ func (ml *ModelLoader) GetWatchDog() *WatchDog { return ml.wd } +func (ml *ModelLoader) BackendLogs() *BackendLogStore { + return ml.backendLogs +} + +func (ml *ModelLoader) SetBackendLoggingEnabled(enabled bool) { + ml.backendLoggingEnabled.Store(enabled) +} + +func (ml *ModelLoader) BackendLoggingEnabled() bool { + return ml.backendLoggingEnabled.Load() +} + // SetLRUEvictionRetrySettings updates the LRU eviction retry settings func (ml *ModelLoader) SetLRUEvictionRetrySettings(maxRetries int, retryInterval time.Duration) { ml.mu.Lock() diff --git a/pkg/model/process.go b/pkg/model/process.go index a1f335afdb8e..df7c3333bbea 100644 --- a/pkg/model/process.go +++ b/pkg/model/process.go @@ -159,19 +159,27 @@ func (ml *ModelLoader) startProcess(grpcProcess, id string, serverAddress string go func() { t, err := tail.TailFile(grpcControlProcess.StderrPath(), tail.Config{Follow: true}) if err != nil { - xlog.Debug("Could not tail stderr") + xlog.Error("Could not tail stderr", "process", grpcProcess) + return } for line := range t.Lines { xlog.Debug("GRPC stderr", "id", strings.Join([]string{id, serverAddress}, "-"), "line", line.Text) + if ml.backendLogs != nil && ml.backendLoggingEnabled.Load() { + ml.backendLogs.AppendLine(id, "stderr", line.Text) + } } }() go func() { t, err := tail.TailFile(grpcControlProcess.StdoutPath(), tail.Config{Follow: true}) if err != nil { - xlog.Debug("Could not tail stdout") + xlog.Error("Could not tail stdout", "process", grpcProcess) + return } for line := range t.Lines { xlog.Debug("GRPC stdout", "id", strings.Join([]string{id, serverAddress}, "-"), "line", line.Text) + if ml.backendLogs != nil && ml.backendLoggingEnabled.Load() { + ml.backendLogs.AppendLine(id, "stdout", line.Text) + } } }() diff --git a/tests/e2e-ui/main.go b/tests/e2e-ui/main.go index d512bce82b1a..7aca8e7e4d69 100644 --- a/tests/e2e-ui/main.go +++ b/tests/e2e-ui/main.go @@ -98,6 +98,7 @@ func main() { config.WithDynamicConfigDir(dataDir), config.WithGeneratedContentDir(generatedDir), config.EnableTracing, + config.EnableBackendLogging, ) if err != nil { fmt.Fprintf(os.Stderr, "error creating application: %v\n", err)