Bugfix/caching

README.md

@@ -81,13 +81,15 @@ lint:
   xunitReport: report.xml
   jsonFile: ""
   ignoreNoqa: false
-  noCache: false
   concurrency: 4
   regoTrace: false
   skip:
     example/doc:
       - rule: "001_002"
         reason: accepted risk
+cache:
+  directory: .mendix-cache/mxlint
+  enable: true
 modelsource: modelsource
 projectDirectory: .
 export:
@@ -103,7 +105,8 @@ Notes:
 - `rules.path` is the local rules directory used by `lint`.
 - `rules.rulesets` are synchronized into `rules.path` before linting.
 - `lint.skip` supports skipping by document path (relative to `modelsource`) and rule number.
-- `lint.noCache` disables lint result cache when set to `true`.
+- `cache.enable` controls lint and export caching. Set to `false` to disable both.
+- `cache.directory` sets the base directory for lint and export cache files.
 - `lint.concurrency` limits how many rules are evaluated in parallel. Lower values reduce peak memory usage for large models.
 - `lint.regoTrace` enables OPA tracing for Rego rules. Keep it `false` for normal runs to reduce memory overhead.
 

default.yaml

@@ -6,7 +6,6 @@ lint:
   xunitReport: ""
   jsonFile: ""
   ignoreNoqa: false
-  noCache: false
   concurrency: 4
   regoTrace: false
   # skip: maps document path (relative to model source, or absolute) to rules to skip.
@@ -21,6 +20,7 @@ export:
   filter: ".*"
   raw: false
   appstore: false
+  concurrency: 4
 serve:
   port: 8082
   debounce: 500

lint/config.go

@@ -60,16 +60,16 @@ func (c *ConfigRulesSpec) UnmarshalYAML(value *yaml.Node) error {
 }
 
 type ConfigExportSpec struct {
-	Filter   string `yaml:"filter"`
-	Raw      *bool  `yaml:"raw"`
-	Appstore *bool  `yaml:"appstore"`
+	Filter      string `yaml:"filter"`
+	Raw         *bool  `yaml:"raw"`
+	Appstore    *bool  `yaml:"appstore"`
+	Concurrency *int   `yaml:"concurrency"`
 }
 
 type ConfigLintSpec struct {
 	XunitReport string                      `yaml:"xunitReport"`
 	JSONFile    string                      `yaml:"jsonFile"`
 	IgnoreNoqa  *bool                       `yaml:"ignoreNoqa"`
-	NoCache     *bool                       `yaml:"noCache"`
 	Concurrency *int                        `yaml:"concurrency"`
 	RegoTrace   *bool                       `yaml:"regoTrace"`
 	Skip        map[string][]ConfigSkipRule `yaml:"skip"`
@@ -311,6 +311,9 @@ func mergeConfig(base *Config, overlay *Config) {
 	if overlay.Export.Appstore != nil {
 		base.Export.Appstore = overlay.Export.Appstore
 	}
+	if overlay.Export.Concurrency != nil {
+		base.Export.Concurrency = overlay.Export.Concurrency
+	}
 
 	if strings.TrimSpace(overlay.Modelsource) != "" {
 		base.Modelsource = strings.TrimSpace(overlay.Modelsource)
@@ -327,9 +330,6 @@ func mergeConfig(base *Config, overlay *Config) {
 	if overlay.Lint.IgnoreNoqa != nil {
 		base.Lint.IgnoreNoqa = overlay.Lint.IgnoreNoqa
 	}
-	if overlay.Lint.NoCache != nil {
-		base.Lint.NoCache = overlay.Lint.NoCache
-	}
 	if overlay.Lint.Concurrency != nil {
 		base.Lint.Concurrency = overlay.Lint.Concurrency
 	}
@@ -344,6 +344,13 @@ func mergeConfig(base *Config, overlay *Config) {
 		base.Serve.Debounce = overlay.Serve.Debounce
 	}
 
+	if strings.TrimSpace(overlay.Cache.Directory) != "" {
+		base.Cache.Directory = strings.TrimSpace(overlay.Cache.Directory)
+	}
+	if overlay.Cache.Enable != nil {
+		base.Cache.Enable = overlay.Cache.Enable
+	}
+
 	if len(overlay.Lint.Skip) == 0 {
 		return
 	}

lint/config_test.go

@@ -390,6 +390,48 @@ func TestLoadMergedConfig_SkipAllDocumentsWildcardKey(t *testing.T) {
 	}
 }
 
+func TestLoadMergedConfig_CacheFromDefaultAndProject(t *testing.T) {
+	projectDir := t.TempDir()
+	enableFalse := false
+	t.Setenv("MXLINT_SYSTEM_CONFIG", filepath.Join(t.TempDir(), "missing-system.yaml"))
+
+	defaultConfig := `cache:
+  directory: .mendix-cache/default-cache
+  enable: true
+`
+	setDefaultConfigForTest(t, defaultConfig)
+
+	cfg, err := LoadMergedConfig(projectDir)
+	if err != nil {
+		t.Fatalf("LoadMergedConfig returned error: %v", err)
+	}
+	if cfg.Cache.Directory != ".mendix-cache/default-cache" {
+		t.Fatalf("expected default cache directory, got %q", cfg.Cache.Directory)
+	}
+	if cfg.Cache.Enable == nil || *cfg.Cache.Enable != true {
+		t.Fatalf("expected default cache.enable=true, got %#v", cfg.Cache.Enable)
+	}
+
+	projectConfig := `cache:
+  directory: .mendix-cache/mxlint
+  enable: false
+`
+	if err := os.WriteFile(filepath.Join(projectDir, "mxlint.yaml"), []byte(projectConfig), 0644); err != nil {
+		t.Fatalf("failed to write project config: %v", err)
+	}
+
+	cfg, err = LoadMergedConfig(projectDir)
+	if err != nil {
+		t.Fatalf("LoadMergedConfig returned error: %v", err)
+	}
+	if cfg.Cache.Directory != ".mendix-cache/mxlint" {
+		t.Fatalf("expected project cache directory override, got %q", cfg.Cache.Directory)
+	}
+	if cfg.Cache.Enable == nil || *cfg.Cache.Enable != enableFalse {
+		t.Fatalf("expected project cache.enable=false, got %#v", cfg.Cache.Enable)
+	}
+}
+
 func TestLoadMergedConfig_LintConcurrencyAndTrace(t *testing.T) {
 	projectDir := t.TempDir()
 	setDefaultConfigForTest(t, "")

main.go

@@ -64,6 +64,7 @@ func main() {
 			mpr.SetLogger(log)
 			lint.SetConfig(config)
 			configureCache(config, projectDir)
+			configureExport(config)
 
 			inputDirectory := config.ProjectDirectory
 			outputDirectory := config.Modelsource
@@ -298,6 +299,14 @@ func main() {
 	}
 }
 
+func configureExport(config *lint.Config) {
+	if config == nil {
+		mpr.ConfigureExportConcurrency(nil)
+		return
+	}
+	mpr.ConfigureExportConcurrency(config.Export.Concurrency)
+}
+
 func configureCache(config *lint.Config, projectDir string) {
 	if config == nil {
 		return
@@ -314,6 +323,7 @@ func configureCache(config *lint.Config, projectDir string) {
 	lint.SetCacheDirectory(filepath.Join(cacheBase, "lint"))
 	mpr.SetPersistentYAMLCacheDirectory(filepath.Join(cacheBase, "mpr-v2-yaml"))
 	mpr.SetPersistentYAMLCacheEnabled(boolValue(config.Cache.Enable, true))
+	mpr.SetExportManifestPath(filepath.Join(cacheBase, "export-manifest.json"))
 }
 
 func boolValue(value *bool, fallback bool) bool {
@@ -327,9 +337,6 @@ func effectiveLintUseCache(config *lint.Config) bool {
 	if config == nil {
 		return true
 	}
-	if boolValue(config.Lint.NoCache, false) {
-		return false
-	}
 	return boolValue(config.Cache.Enable, true)
 }
 

main_test.go

@@ -8,7 +8,6 @@ import (
 
 func TestEffectiveLintUseCache(t *testing.T) {
 	falseValue := false
-	trueValue := true
 
 	tests := []struct {
 		name     string
@@ -26,9 +25,6 @@ func TestEffectiveLintUseCache(t *testing.T) {
 				Cache: lint.ConfigCacheSpec{
 					Enable: nil,
 				},
-				Lint: lint.ConfigLintSpec{
-					NoCache: nil,
-				},
 			},
 			expected: true,
 		},
@@ -41,18 +37,6 @@ func TestEffectiveLintUseCache(t *testing.T) {
 			},
 			expected: false,
 		},
-		{
-			name: "cache disabled by lint.noCache even when cache.enable true",
-			config: &lint.Config{
-				Cache: lint.ConfigCacheSpec{
-					Enable: &trueValue,
-				},
-				Lint: lint.ConfigLintSpec{
-					NoCache: &trueValue,
-				},
-			},
-			expected: false,
-		},
 	}
 
 	for _, tt := range tests {

mpr/export_manifest.go

@@ -0,0 +1,132 @@
+package mpr
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+)
+
+const exportManifestVersion = 1
+
+type exportManifestEntry struct {
+	Name         string `json:"name"`
+	Type         string `json:"type"`
+	FolderPath   string `json:"folderPath"`
+	RelativePath string `json:"relativePath"`
+	ContentsHash string `json:"contentsHash"`
+	ModTimeNs    int64  `json:"modTimeNs,omitempty"`
+	FileSize     int64  `json:"fileSize,omitempty"`
+}
+
+type exportManifest struct {
+	Version int                            `json:"version"`
+	Entries map[string]exportManifestEntry `json:"entries"`
+}
+
+var exportManifestSettings = struct {
+	mu   sync.RWMutex
+	path string
+}{
+	path: "",
+}
+
+func SetExportManifestPath(path string) {
+	exportManifestSettings.mu.Lock()
+	defer exportManifestSettings.mu.Unlock()
+	exportManifestSettings.path = strings.TrimSpace(path)
+}
+
+func getExportManifestPath() string {
+	exportManifestSettings.mu.RLock()
+	defer exportManifestSettings.mu.RUnlock()
+	return exportManifestSettings.path
+}
+
+func newExportManifest() *exportManifest {
+	return &exportManifest{
+		Version: exportManifestVersion,
+		Entries: make(map[string]exportManifestEntry),
+	}
+}
+
+func loadExportManifest(path string) (*exportManifest, error) {
+	if path == "" {
+		return newExportManifest(), nil
+	}
+	data, err := os.ReadFile(path)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return newExportManifest(), nil
+		}
+		return nil, fmt.Errorf("error reading export manifest: %w", err)
+	}
+	manifest := &exportManifest{}
+	if err := json.Unmarshal(data, manifest); err != nil {
+		return nil, fmt.Errorf("error parsing export manifest: %w", err)
+	}
+	if manifest.Entries == nil {
+		manifest.Entries = make(map[string]exportManifestEntry)
+	}
+	return manifest, nil
+}
+
+func saveExportManifest(path string, manifest *exportManifest) error {
+	if path == "" || manifest == nil {
+		return nil
+	}
+	if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
+		return fmt.Errorf("error creating export manifest directory: %w", err)
+	}
+	data, err := json.MarshalIndent(manifest, "", "  ")
+	if err != nil {
+		return fmt.Errorf("error marshaling export manifest: %w", err)
+	}
+	tmpPath := path + ".tmp"
+	if err := os.WriteFile(tmpPath, data, 0644); err != nil {
+		return fmt.Errorf("error writing export manifest: %w", err)
+	}
+	if err := os.Rename(tmpPath, path); err != nil {
+		return fmt.Errorf("error saving export manifest: %w", err)
+	}
+	return nil
+}
+
+func (m *exportManifest) entryFor(unitID, contentsHash string) (exportManifestEntry, bool) {
+	if m == nil {
+		return exportManifestEntry{}, false
+	}
+	entry, ok := m.Entries[unitID]
+	if !ok || entry.ContentsHash != contentsHash {
+		return exportManifestEntry{}, false
+	}
+	return entry, true
+}
+
+func mxunitFileStat(path string) (modTimeNs, size int64, err error) {
+	info, err := os.Stat(path)
+	if err != nil {
+		return 0, 0, err
+	}
+	return info.ModTime().UnixNano(), info.Size(), nil
+}
+
+// manifestFastPathHint returns true when mtime and size match the manifest entry.
+func manifestFastPathHint(entry exportManifestEntry, mxunitPath string) bool {
+	if entry.ModTimeNs == 0 && entry.FileSize == 0 {
+		return true
+	}
+	modTimeNs, size, err := mxunitFileStat(mxunitPath)
+	if err != nil {
+		return false
+	}
+	if entry.FileSize != 0 && entry.FileSize != size {
+		return false
+	}
+	if entry.ModTimeNs != 0 && entry.ModTimeNs != modTimeNs {
+		return false
+	}
+	return true
+}