feat(similarity): 脚本相似度检测与完整性校验系统

.gitignore

@@ -10,6 +10,7 @@
 
 .claude/settings.local.json
 CLAUDE.md
+.omc
 
 # ip2region data files (download separately)
 /data/*.xdb

.golangci.yml

@@ -30,6 +30,10 @@ linters:
     misspell:
       locale: US
 
+  exclusions:
+    paths:
+      - docs
+
 formatters:
   enable:
     - gofmt

cmd/app/main.go

@@ -30,10 +30,13 @@ import (
 	"github.com/scriptscat/scriptlist/internal/repository/report_repo"
 	"github.com/scriptscat/scriptlist/internal/repository/resource_repo"
 	"github.com/scriptscat/scriptlist/internal/repository/script_repo"
+	"github.com/scriptscat/scriptlist/internal/repository/similarity_repo"
 	"github.com/scriptscat/scriptlist/internal/repository/statistics_repo"
 	"github.com/scriptscat/scriptlist/internal/repository/user_repo"
+	"github.com/scriptscat/scriptlist/internal/service/similarity_svc"
 	"github.com/scriptscat/scriptlist/internal/task/consumer"
 	"github.com/scriptscat/scriptlist/internal/task/crontab"
+	"github.com/scriptscat/scriptlist/internal/task/crontab/handler"
 	"github.com/scriptscat/scriptlist/migrations"
 )
 
@@ -103,6 +106,24 @@ func main() {
 
 	announcement_repo.RegisterAnnouncement(announcement_repo.NewAnnouncement())
 
+	// similarity / integrity
+	similarity_repo.RegisterFingerprint(similarity_repo.NewFingerprintRepo())
+	similarity_repo.RegisterSimilarPair(similarity_repo.NewSimilarPairRepo())
+	similarity_repo.RegisterSuspectSummary(similarity_repo.NewSuspectSummaryRepo())
+	similarity_repo.RegisterSimilarityWhitelist(similarity_repo.NewSimilarityWhitelistRepo())
+	similarity_repo.RegisterIntegrityWhitelist(similarity_repo.NewIntegrityWhitelistRepo())
+	similarity_repo.RegisterIntegrityReview(similarity_repo.NewIntegrityReviewRepo())
+	similarity_repo.RegisterFingerprintES(similarity_repo.NewFingerprintESRepo())
+	similarity_repo.RegisterPatrolQuery(similarity_repo.NewPatrolQueryRepo())
+	similarity_svc.RegisterIntegrity(similarity_svc.NewIntegritySvc())
+	similarity_svc.RegisterScan(similarity_svc.NewScanSvc())
+	similarity_svc.RegisterAccess(similarity_svc.NewAccessSvc())
+	similarity_svc.RegisterAdmin(similarity_svc.NewAdminSvc())
+	// Wire the crontab handlers' long-running methods into admin_svc so
+	// the admin endpoints can invoke them without an import cycle.
+	similarity_svc.RegisterBackfillRunner(handler.NewSimilarityPatrolHandler().RunBackfill)
+	similarity_svc.RegisterStopFpRefresher(handler.NewSimilarityStopFpHandler().Refresh)
+
 	err = cago.New(ctx, cfg).
 		Registry(component.Core()).
 		Registry(db.Database()).
@@ -118,6 +139,12 @@ func main() {
 			return nil
 		})).
 		Registry(cago.FuncComponent(appconfigs.Validate)).
+		Registry(cago.FuncComponent(func(ctx context.Context, cfg *configs.Config) error {
+			if !appconfigs.Similarity().ScanEnabled {
+				return nil
+			}
+			return similarity_repo.EnsureFingerprintIndex(ctx)
+		})).
 		Registry(cago.FuncComponent(func(ctx context.Context, cfg *configs.Config) error {
 			v4Path := cfg.String(ctx, "ip2region.v4_xdb_path")
 			v6Path := cfg.String(ctx, "ip2region.v6_xdb_path")

configs/config.go

@@ -101,11 +101,148 @@ func QQMigrate() *QQMigrateConfig {
 	return cfg
 }
 
+// SimilarityConfig 相似度检测系统配置（YAML + DB 覆盖）
+type SimilarityConfig struct {
+	ScanEnabled               bool    `yaml:"scan_enabled"`
+	JaccardThreshold          float64 `yaml:"jaccard_threshold"`
+	CoverageThreshold         float64 `yaml:"coverage_threshold"`
+	KGramSize                 int     `yaml:"kgram_size"`
+	WinnowingWindow           int     `yaml:"winnowing_window"`
+	MinFingerprints           int     `yaml:"min_fingerprints"`
+	MaxCodeSize               int     `yaml:"max_code_size"`
+	StopFpDfCutoff            int     `yaml:"stop_fp_df_cutoff"`
+	StopFpRefreshSec          int     `yaml:"stop_fp_refresh_sec"`
+	BackfillBatchSize         int     `yaml:"backfill_batch_size"`
+	BackfillSleepMs           int     `yaml:"backfill_sleep_ms"`
+	IntegrityEnabled          bool    `yaml:"integrity_enabled"`
+	IntegrityWarnThreshold    float64 `yaml:"integrity_warn_threshold"`
+	IntegrityBlockThreshold   float64 `yaml:"integrity_block_threshold"`
+	IntegrityAsyncAutoArchive *bool   `yaml:"integrity_async_auto_archive"`
+}
+
+// Similarity 返回相似度系统配置。读取顺序：
+//  1. YAML 文件（configs/config.yaml）— 进程启动时的基线
+//  2. spec §6.1 默认值 — 填补 YAML 未声明字段
+//  3. pre_system_config 表的 `similarity.*` 行 — 管理员后台动态覆盖
+//
+// DB override 允许管理员在不重启服务的情况下调整阈值 / 开关，符合 spec §1.1
+// "可配置阈值：管理员可动态调整相似度阈值、覆盖率阈值、完整性检查阈值"。
+// DB 行写入失败或解析失败时静默回退到 YAML 值。
+func Similarity() *SimilarityConfig {
+	// 预填 spec §6.1 的 bool 默认值——YAML Scan 只会覆盖被显式声明的字段，
+	// 所以即使 YAML 完全省略 similarity 段，bool 仍是 true（spec 默认）。
+	cfg := &SimilarityConfig{
+		ScanEnabled:      true,
+		IntegrityEnabled: true,
+	}
+	if d := configs.Default(); d != nil {
+		_ = d.Scan(context.Background(), "similarity", cfg)
+	}
+	// Apply defaults to any zero-valued field (zero is sentinel for "unset" here).
+	if cfg.JaccardThreshold == 0 {
+		cfg.JaccardThreshold = 0.30
+	}
+	if cfg.CoverageThreshold == 0 {
+		cfg.CoverageThreshold = 0.50
+	}
+	if cfg.KGramSize == 0 {
+		cfg.KGramSize = 5
+	}
+	if cfg.WinnowingWindow == 0 {
+		cfg.WinnowingWindow = 10
+	}
+	if cfg.MinFingerprints == 0 {
+		cfg.MinFingerprints = 20
+	}
+	// MaxCodeSize: 0 means unlimited (subject to the API-level 10MB cap on
+	// Code). scan.go gates on `MaxCodeSize > 0` so zero disables the guard.
+	if cfg.StopFpDfCutoff == 0 {
+		cfg.StopFpDfCutoff = 50
+	}
+	if cfg.StopFpRefreshSec == 0 {
+		cfg.StopFpRefreshSec = 3600
+	}
+	if cfg.BackfillBatchSize == 0 {
+		cfg.BackfillBatchSize = 50
+	}
+	if cfg.BackfillSleepMs == 0 {
+		cfg.BackfillSleepMs = 200
+	}
+	if cfg.IntegrityWarnThreshold == 0 {
+		cfg.IntegrityWarnThreshold = 0.5
+	}
+	if cfg.IntegrityBlockThreshold == 0 {
+		cfg.IntegrityBlockThreshold = 0.8
+	}
+	if cfg.IntegrityAsyncAutoArchive == nil {
+		t := true
+		cfg.IntegrityAsyncAutoArchive = &t
+	}
+	// DB overrides (admin-tunable at runtime per spec §1.1 / §6.1).
+	if dbProvider != nil {
+		ctx := context.Background()
+		if v, ok := dbProvider.GetBool(ctx, "similarity.scan_enabled"); ok {
+			cfg.ScanEnabled = v
+		}
+		if v, ok := dbProvider.GetFloat(ctx, "similarity.jaccard_threshold"); ok {
+			cfg.JaccardThreshold = v
+		}
+		if v, ok := dbProvider.GetFloat(ctx, "similarity.coverage_threshold"); ok {
+			cfg.CoverageThreshold = v
+		}
+		if v, ok := dbProvider.GetInt(ctx, "similarity.kgram_size"); ok {
+			cfg.KGramSize = v
+		}
+		if v, ok := dbProvider.GetInt(ctx, "similarity.winnowing_window"); ok {
+			cfg.WinnowingWindow = v
+		}
+		if v, ok := dbProvider.GetInt(ctx, "similarity.min_fingerprints"); ok {
+			cfg.MinFingerprints = v
+		}
+		if v, ok := dbProvider.GetInt(ctx, "similarity.max_code_size"); ok {
+			cfg.MaxCodeSize = v
+		}
+		if v, ok := dbProvider.GetInt(ctx, "similarity.stop_fp_df_cutoff"); ok {
+			cfg.StopFpDfCutoff = v
+		}
+		if v, ok := dbProvider.GetInt(ctx, "similarity.stop_fp_refresh_sec"); ok {
+			cfg.StopFpRefreshSec = v
+		}
+		if v, ok := dbProvider.GetInt(ctx, "similarity.backfill_batch_size"); ok {
+			cfg.BackfillBatchSize = v
+		}
+		if v, ok := dbProvider.GetInt(ctx, "similarity.backfill_sleep_ms"); ok {
+			cfg.BackfillSleepMs = v
+		}
+		if v, ok := dbProvider.GetBool(ctx, "similarity.integrity_enabled"); ok {
+			cfg.IntegrityEnabled = v
+		}
+		if v, ok := dbProvider.GetFloat(ctx, "similarity.integrity_warn_threshold"); ok {
+			cfg.IntegrityWarnThreshold = v
+		}
+		if v, ok := dbProvider.GetFloat(ctx, "similarity.integrity_block_threshold"); ok {
+			cfg.IntegrityBlockThreshold = v
+		}
+		if v, ok := dbProvider.GetBool(ctx, "similarity.integrity_async_auto_archive"); ok {
+			cfg.IntegrityAsyncAutoArchive = &v
+		}
+	}
+	return cfg
+}
+
 // Validate 在服务启动时检查必要配置项（符合 CaGo FuncComponent 签名）
 // 其余配置（turnstile、ai）可通过管理后台动态配置，不在启动时强制校验
 func Validate(ctx context.Context, cfg *configs.Config) error {
 	if cfg.String(ctx, "website.url") == "" {
 		return fmt.Errorf("missing required config key: website.url")
 	}
+	// similarity.scan_enabled=true 需要 elasticsearch 地址（cago 读取 elasticsearch.address 列表）
+	if Similarity().ScanEnabled {
+		var esAddress []string
+		_ = cfg.Scan(ctx, "elasticsearch.address", &esAddress)
+		if len(esAddress) == 0 {
+			return fmt.Errorf("similarity.scan_enabled=true requires elasticsearch.address to be set")
+		}
+	}
 	return nil
 }

configs/config.yaml.example

@@ -50,5 +50,21 @@ qq_migrate:
 ip2region:
   v4_xdb_path: "data/ip2region_v4.xdb"
   v6_xdb_path: "data/ip2region_v6.xdb"
+similarity:
+  scan_enabled: true
+  jaccard_threshold: 0.30
+  coverage_threshold: 0.50
+  kgram_size: 5
+  winnowing_window: 10
+  min_fingerprints: 20
+  max_code_size: 0  # 0 = unlimited (API already caps Code at 10MB)
+  stop_fp_df_cutoff: 50
+  stop_fp_refresh_sec: 3600
+  backfill_batch_size: 50
+  backfill_sleep_ms: 200
+  integrity_enabled: true
+  integrity_warn_threshold: 0.5
+  integrity_block_threshold: 0.8
+  integrity_async_auto_archive: true
 source: file
 version: 2.0.0

configs/db_provider.go

@@ -2,6 +2,7 @@ package configs
 
 import (
 	"context"
+	"strconv"
 
 	"github.com/scriptscat/scriptlist/internal/repository/system_config_repo"
 )
@@ -25,6 +26,51 @@ func (p *DBConfigProvider) GetString(ctx context.Context, key string) (string, b
 	return cfg.ConfigValue, true
 }
 
+// GetBool returns the DB-stored value for key as a bool. Recognized truthy
+// literals: "true", "1", "yes", "on" (case-insensitive). Missing rows and
+// parse failures return (false, false) so callers can fall back to YAML.
+func (p *DBConfigProvider) GetBool(ctx context.Context, key string) (bool, bool) {
+	raw, ok := p.GetString(ctx, key)
+	if !ok {
+		return false, false
+	}
+	switch raw {
+	case "true", "True", "TRUE", "1", "yes", "Yes", "YES", "on", "On", "ON":
+		return true, true
+	case "false", "False", "FALSE", "0", "no", "No", "NO", "off", "Off", "OFF":
+		return false, true
+	}
+	return false, false
+}
+
+// GetFloat returns the DB-stored value for key as a float64. Missing rows
+// and parse failures return (0, false).
+func (p *DBConfigProvider) GetFloat(ctx context.Context, key string) (float64, bool) {
+	raw, ok := p.GetString(ctx, key)
+	if !ok {
+		return 0, false
+	}
+	v, err := strconv.ParseFloat(raw, 64)
+	if err != nil {
+		return 0, false
+	}
+	return v, true
+}
+
+// GetInt returns the DB-stored value for key as an int. Missing rows and
+// parse failures return (0, false).
+func (p *DBConfigProvider) GetInt(ctx context.Context, key string) (int, bool) {
+	raw, ok := p.GetString(ctx, key)
+	if !ok {
+		return 0, false
+	}
+	v, err := strconv.Atoi(raw)
+	if err != nil {
+		return 0, false
+	}
+	return v, true
+}
+
 func (p *DBConfigProvider) GetByPrefix(ctx context.Context, prefix string) (map[string]string, error) {
 	repo := system_config_repo.SystemConfig()
 	if repo == nil {