feat(ui): Per model backend logs and various fixes

core/application/startup.go

@@ -136,6 +136,8 @@ func New(opts ...config.AppOption) (*Application, error) {
 		loadRuntimeSettingsFromFile(options)
 	}
 
+	application.ModelLoader().SetBackendLoggingEnabled(options.EnableBackendLogging)
+
 	// turn off any process that was started by GRPC if the context is canceled
 	go func() {
 		<-options.Context.Done()
@@ -382,6 +384,12 @@ func loadRuntimeSettingsFromFile(options *config.ApplicationConfig) {
 		}
 	}
 
+	if settings.EnableBackendLogging != nil {
+		if !options.EnableBackendLogging {
+			options.EnableBackendLogging = *settings.EnableBackendLogging
+		}
+	}
+
 	xlog.Debug("Runtime settings loaded from runtime_settings.json")
 }
 

core/backend/detection.go

@@ -3,8 +3,10 @@ package backend
 import (
 	"context"
 	"fmt"
+	"time"
 
 	"github.com/mudler/LocalAI/core/config"
+	"github.com/mudler/LocalAI/core/trace"
 	"github.com/mudler/LocalAI/pkg/grpc/proto"
 	"github.com/mudler/LocalAI/pkg/model"
 )
@@ -18,16 +20,43 @@ func Detection(
 	opts := ModelOptions(modelConfig, appConfig)
 	detectionModel, err := loader.Load(opts...)
 	if err != nil {
+		recordModelLoadFailure(appConfig, modelConfig.Name, modelConfig.Backend, err, nil)
 		return nil, err
 	}
 
 	if detectionModel == nil {
 		return nil, fmt.Errorf("could not load detection model")
 	}
 
+	var startTime time.Time
+	if appConfig.EnableTracing {
+		trace.InitBackendTracingIfEnabled(appConfig.TracingMaxItems)
+		startTime = time.Now()
+	}
+
 	res, err := detectionModel.Detect(context.Background(), &proto.DetectOptions{
 		Src: sourceFile,
 	})
 
+	if appConfig.EnableTracing {
+		errStr := ""
+		if err != nil {
+			errStr = err.Error()
+		}
+
+		trace.RecordBackendTrace(trace.BackendTrace{
+			Timestamp: startTime,
+			Duration:  time.Since(startTime),
+			Type:      trace.BackendTraceDetection,
+			ModelName: modelConfig.Name,
+			Backend:   modelConfig.Backend,
+			Summary:   trace.TruncateString(sourceFile, 200),
+			Error:     errStr,
+			Data: map[string]any{
+				"source_file": sourceFile,
+			},
+		})
+	}
+
 	return res, err
 }

core/backend/embeddings.go

@@ -17,6 +17,7 @@ func ModelEmbedding(s string, tokens []int, loader *model.ModelLoader, modelConf
 
 	inferenceModel, err := loader.Load(opts...)
 	if err != nil {
+		recordModelLoadFailure(appConfig, modelConfig.Name, modelConfig.Backend, err, nil)
 		return nil, err
 	}
 

core/backend/image.go

@@ -17,6 +17,7 @@ func ImageGeneration(height, width, step, seed int, positive_prompt, negative_pr
 		opts...,
 	)
 	if err != nil {
+		recordModelLoadFailure(appConfig, modelConfig.Name, modelConfig.Backend, err, nil)
 		return nil, err
 	}
 

core/backend/llm.go

@@ -65,6 +65,7 @@ func ModelInference(ctx context.Context, s string, messages schema.Messages, ima
 	opts := ModelOptions(*c, o)
 	inferenceModel, err := loader.Load(opts...)
 	if err != nil {
+		recordModelLoadFailure(o, c.Name, c.Backend, err, map[string]any{"model_file": modelFile})
 		return nil, err
 	}
 

core/backend/options.go

@@ -1,17 +1,36 @@
 package backend
 
 import (
-	"strings"
 	"math/rand"
 	"os"
 	"path/filepath"
+	"strings"
+	"time"
 
 	"github.com/mudler/LocalAI/core/config"
+	"github.com/mudler/LocalAI/core/trace"
 	pb "github.com/mudler/LocalAI/pkg/grpc/proto"
 	"github.com/mudler/LocalAI/pkg/model"
 	"github.com/mudler/xlog"
 )
 
+// recordModelLoadFailure records a backend trace when model loading fails.
+func recordModelLoadFailure(appConfig *config.ApplicationConfig, modelName, backend string, err error, data map[string]any) {
+	if !appConfig.EnableTracing {
+		return
+	}
+	trace.InitBackendTracingIfEnabled(appConfig.TracingMaxItems)
+	trace.RecordBackendTrace(trace.BackendTrace{
+		Timestamp: time.Now(),
+		Type:      trace.BackendTraceModelLoad,
+		ModelName: modelName,
+		Backend:   backend,
+		Summary:   "Model load failed",
+		Error:     err.Error(),
+		Data:      data,
+	})
+}
+
 func ModelOptions(c config.ModelConfig, so *config.ApplicationConfig, opts ...model.Option) []model.Option {
 	name := c.Name
 	if name == "" {

core/backend/rerank.go

@@ -15,6 +15,7 @@ func Rerank(request *proto.RerankRequest, loader *model.ModelLoader, appConfig *
 	opts := ModelOptions(modelConfig, appConfig)
 	rerankModel, err := loader.Load(opts...)
 	if err != nil {
+		recordModelLoadFailure(appConfig, modelConfig.Name, modelConfig.Backend, err, nil)
 		return nil, err
 	}
 

core/backend/soundgeneration.go

@@ -37,6 +37,7 @@ func SoundGeneration(
 	opts := ModelOptions(modelConfig, appConfig)
 	soundGenModel, err := loader.Load(opts...)
 	if err != nil {
+		recordModelLoadFailure(appConfig, modelConfig.Name, modelConfig.Backend, err, nil)
 		return "", nil, err
 	}
 

core/backend/token_metrics.go

@@ -18,6 +18,7 @@ func TokenMetrics(
 	opts := ModelOptions(modelConfig, appConfig, model.WithModel(modelFile))
 	model, err := loader.Load(opts...)
 	if err != nil {
+		recordModelLoadFailure(appConfig, modelConfig.Name, modelConfig.Backend, err, nil)
 		return nil, err
 	}
 

core/backend/tokenize.go

@@ -18,6 +18,7 @@ func ModelTokenize(s string, loader *model.ModelLoader, modelConfig config.Model
 	opts := ModelOptions(modelConfig, appConfig)
 	inferenceModel, err = loader.Load(opts...)
 	if err != nil {
+		recordModelLoadFailure(appConfig, modelConfig.Name, modelConfig.Backend, err, nil)
 		return schema.TokenizeResponse{}, err
 	}
 

core/backend/transcript.go

@@ -23,6 +23,7 @@ func ModelTranscription(audio, language string, translate, diarize bool, prompt
 
 	transcriptionModel, err := ml.Load(opts...)
 	if err != nil {
+		recordModelLoadFailure(appConfig, modelConfig.Name, modelConfig.Backend, err, nil)
 		return nil, err
 	}
 

core/backend/tts.go

@@ -31,6 +31,7 @@ func ModelTTS(
 	opts := ModelOptions(modelConfig, appConfig)
 	ttsModel, err := loader.Load(opts...)
 	if err != nil {
+		recordModelLoadFailure(appConfig, modelConfig.Name, modelConfig.Backend, err, nil)
 		return "", nil, err
 	}
 
@@ -131,6 +132,7 @@ func ModelTTSStream(
 	opts := ModelOptions(modelConfig, appConfig)
 	ttsModel, err := loader.Load(opts...)
 	if err != nil {
+		recordModelLoadFailure(appConfig, modelConfig.Name, modelConfig.Backend, err, nil)
 		return err
 	}
 

core/backend/vad.go

@@ -17,6 +17,7 @@ func VAD(request *schema.VADRequest,
 	opts := ModelOptions(modelConfig, appConfig)
 	vadModel, err := ml.Load(opts...)
 	if err != nil {
+		recordModelLoadFailure(appConfig, modelConfig.Name, modelConfig.Backend, err, nil)
 		return nil, err
 	}
 

core/backend/video.go

@@ -17,6 +17,7 @@ func VideoGeneration(height, width int32, prompt, negativePrompt, startImage, en
 		opts...,
 	)
 	if err != nil {
+		recordModelLoadFailure(appConfig, modelConfig.Name, modelConfig.Backend, err, nil)
 		return nil, err
 	}
 

core/config/application_config.go

@@ -21,6 +21,7 @@ type ApplicationConfig struct {
 	Debug                               bool
 	EnableTracing                       bool
 	TracingMaxItems                     int
+	EnableBackendLogging                bool
 	GeneratedContentDir                 string
 
 	UploadDir string
@@ -213,6 +214,10 @@ var EnableTracing = func(o *ApplicationConfig) {
 	o.EnableTracing = true
 }
 
+var EnableBackendLogging = func(o *ApplicationConfig) {
+	o.EnableBackendLogging = true
+}
+
 var EnableWatchDogIdleCheck = func(o *ApplicationConfig) {
 	o.WatchDog = true
 	o.WatchDogIdle = true
@@ -743,6 +748,7 @@ func (o *ApplicationConfig) ToRuntimeSettings() RuntimeSettings {
 	debug := o.Debug
 	tracingMaxItems := o.TracingMaxItems
 	enableTracing := o.EnableTracing
+	enableBackendLogging := o.EnableBackendLogging
 	cors := o.CORS
 	csrf := o.CSRF
 	corsAllowOrigins := o.CORSAllowOrigins
@@ -816,6 +822,7 @@ func (o *ApplicationConfig) ToRuntimeSettings() RuntimeSettings {
 		Debug:                    &debug,
 		TracingMaxItems:          &tracingMaxItems,
 		EnableTracing:            &enableTracing,
+		EnableBackendLogging:     &enableBackendLogging,
 		CORS:                     &cors,
 		CSRF:                     &csrf,
 		CORSAllowOrigins:         &corsAllowOrigins,
@@ -944,6 +951,9 @@ func (o *ApplicationConfig) ApplyRuntimeSettings(settings *RuntimeSettings) (req
 	if settings.TracingMaxItems != nil {
 		o.TracingMaxItems = *settings.TracingMaxItems
 	}
+	if settings.EnableBackendLogging != nil {
+		o.EnableBackendLogging = *settings.EnableBackendLogging
+	}
 	if settings.CORS != nil {
 		o.CORS = *settings.CORS
 	}

core/config/runtime_settings.go

@@ -36,8 +36,9 @@ type RuntimeSettings struct {
 	ContextSize     *int  `json:"context_size,omitempty"`
 	F16             *bool `json:"f16,omitempty"`
 	Debug           *bool `json:"debug,omitempty"`
-	EnableTracing   *bool `json:"enable_tracing,omitempty"`
-	TracingMaxItems *int  `json:"tracing_max_items,omitempty"`
+	EnableTracing        *bool `json:"enable_tracing,omitempty"`
+	TracingMaxItems      *int  `json:"tracing_max_items,omitempty"`
+	EnableBackendLogging *bool `json:"enable_backend_logging,omitempty"`
 
 	// Security/CORS settings
 	CORS             *bool   `json:"cors,omitempty"`

core/http/endpoints/localai/settings.go

@@ -136,6 +136,12 @@ func UpdateSettingsEndpoint(app *application.Application) echo.HandlerFunc {
 			appConfig.ApiKeys = append(envKeys, runtimeKeys...)
 		}
 
+		// Update backend logging dynamically
+		if settings.EnableBackendLogging != nil {
+			app.ModelLoader().SetBackendLoggingEnabled(*settings.EnableBackendLogging)
+			xlog.Info("Updated backend logging setting", "enableBackendLogging", *settings.EnableBackendLogging)
+		}
+
 		// Update watchdog dynamically for settings that don't require restart
 		if settings.ForceEvictionWhenBusy != nil {
 			currentWD := app.ModelLoader().GetWatchDog()

core/http/middleware/trace.go

@@ -29,8 +29,10 @@ type APIExchangeResponse struct {
 
 type APIExchange struct {
 	Timestamp time.Time           `json:"timestamp"`
+	Duration  time.Duration       `json:"duration"`
 	Request   APIExchangeRequest  `json:"request"`
 	Response  APIExchangeResponse `json:"response"`
+	Error     string              `json:"error,omitempty"`
 }
 
 var traceBuffer *circularbuffer.Queue[APIExchange]
@@ -108,13 +110,18 @@ func TraceMiddleware(app *application.Application) echo.MiddlewareFunc {
 			}
 			c.Response().Writer = mw
 
-			err = next(c)
-			if err != nil {
-				c.Response().Writer = mw.ResponseWriter // Restore original writer if error
-				return err
+			handlerErr := next(c)
+
+			// Restore original writer unconditionally
+			c.Response().Writer = mw.ResponseWriter
+
+			// Determine response status (use 500 if handler errored and no status was set)
+			status := c.Response().Status
+			if status == 0 && handlerErr != nil {
+				status = http.StatusInternalServerError
 			}
 
-			// Create exchange log
+			// Create exchange log (always, even on error)
 			requestHeaders := c.Request().Header.Clone()
 			requestBody := make([]byte, len(body))
 			copy(requestBody, body)
@@ -123,26 +130,30 @@ func TraceMiddleware(app *application.Application) echo.MiddlewareFunc {
 			copy(responseBody, resBody.Bytes())
 			exchange := APIExchange{
 				Timestamp: startTime,
+				Duration:  time.Since(startTime),
 				Request: APIExchangeRequest{
 					Method:  c.Request().Method,
 					Path:    c.Path(),
 					Headers: &requestHeaders,
 					Body:    &requestBody,
 				},
 				Response: APIExchangeResponse{
-					Status:  c.Response().Status,
+					Status:  status,
 					Headers: &responseHeaders,
 					Body:    &responseBody,
 				},
 			}
+			if handlerErr != nil {
+				exchange.Error = handlerErr.Error()
+			}
 
 			select {
 			case logChan <- exchange:
 			default:
 				xlog.Warn("Trace channel full, dropping trace")
 			}
 
-			return nil
+			return handlerErr
 		}
 	}
 }