From 50122537aa6341affd2a1a39166bfbe66ed4ea8b Mon Sep 17 00:00:00 2001 From: Xiwen Cheng Date: Tue, 7 Jul 2026 23:16:46 +0200 Subject: [PATCH 1/5] Allow caching options to be overriden from yaml config --- lint/config.go | 7 +++++++ lint/config_test.go | 42 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/lint/config.go b/lint/config.go index 3bad402..370f160 100644 --- a/lint/config.go +++ b/lint/config.go @@ -344,6 +344,13 @@ func mergeConfig(base *Config, overlay *Config) { base.Serve.Debounce = overlay.Serve.Debounce } + if strings.TrimSpace(overlay.Cache.Directory) != "" { + base.Cache.Directory = strings.TrimSpace(overlay.Cache.Directory) + } + if overlay.Cache.Enable != nil { + base.Cache.Enable = overlay.Cache.Enable + } + if len(overlay.Lint.Skip) == 0 { return } diff --git a/lint/config_test.go b/lint/config_test.go index 16b6810..86c8481 100644 --- a/lint/config_test.go +++ b/lint/config_test.go @@ -390,6 +390,48 @@ func TestLoadMergedConfig_SkipAllDocumentsWildcardKey(t *testing.T) { } } +func TestLoadMergedConfig_CacheFromDefaultAndProject(t *testing.T) { + projectDir := t.TempDir() + enableFalse := false + t.Setenv("MXLINT_SYSTEM_CONFIG", filepath.Join(t.TempDir(), "missing-system.yaml")) + + defaultConfig := `cache: + directory: .mendix-cache/default-cache + enable: true +` + setDefaultConfigForTest(t, defaultConfig) + + cfg, err := LoadMergedConfig(projectDir) + if err != nil { + t.Fatalf("LoadMergedConfig returned error: %v", err) + } + if cfg.Cache.Directory != ".mendix-cache/default-cache" { + t.Fatalf("expected default cache directory, got %q", cfg.Cache.Directory) + } + if cfg.Cache.Enable == nil || *cfg.Cache.Enable != true { + t.Fatalf("expected default cache.enable=true, got %#v", cfg.Cache.Enable) + } + + projectConfig := `cache: + directory: .mendix-cache/mxlint + enable: false +` + if err := os.WriteFile(filepath.Join(projectDir, "mxlint.yaml"), []byte(projectConfig), 0644); err != nil { + t.Fatalf("failed to write project config: %v", err) + } + + cfg, err = LoadMergedConfig(projectDir) + if err != nil { + t.Fatalf("LoadMergedConfig returned error: %v", err) + } + if cfg.Cache.Directory != ".mendix-cache/mxlint" { + t.Fatalf("expected project cache directory override, got %q", cfg.Cache.Directory) + } + if cfg.Cache.Enable == nil || *cfg.Cache.Enable != enableFalse { + t.Fatalf("expected project cache.enable=false, got %#v", cfg.Cache.Enable) + } +} + func TestLoadMergedConfig_LintConcurrencyAndTrace(t *testing.T) { projectDir := t.TempDir() setDefaultConfigForTest(t, "") From 18aeaae0775de645733f8b1e186f7ab062587d95 Mon Sep 17 00:00:00 2001 From: Xiwen Cheng Date: Tue, 7 Jul 2026 23:32:48 +0200 Subject: [PATCH 2/5] remove lint.noCache option --- README.md | 7 +++++-- default.yaml | 1 - lint/config.go | 4 ---- main.go | 3 --- main_test.go | 16 ---------------- serve/serve.go | 3 --- serve/serve_config_test.go | 14 +++++--------- 7 files changed, 10 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index d6e4ec7..c7cc4af 100644 --- a/README.md +++ b/README.md @@ -81,13 +81,15 @@ lint: xunitReport: report.xml jsonFile: "" ignoreNoqa: false - noCache: false concurrency: 4 regoTrace: false skip: example/doc: - rule: "001_002" reason: accepted risk +cache: + directory: .mendix-cache/mxlint + enable: true modelsource: modelsource projectDirectory: . export: @@ -103,7 +105,8 @@ Notes: - `rules.path` is the local rules directory used by `lint`. - `rules.rulesets` are synchronized into `rules.path` before linting. - `lint.skip` supports skipping by document path (relative to `modelsource`) and rule number. -- `lint.noCache` disables lint result cache when set to `true`. +- `cache.enable` controls lint and export caching. Set to `false` to disable both. +- `cache.directory` sets the base directory for lint and export cache files. - `lint.concurrency` limits how many rules are evaluated in parallel. Lower values reduce peak memory usage for large models. - `lint.regoTrace` enables OPA tracing for Rego rules. Keep it `false` for normal runs to reduce memory overhead. diff --git a/default.yaml b/default.yaml index 3e53129..c7c82cb 100644 --- a/default.yaml +++ b/default.yaml @@ -6,7 +6,6 @@ lint: xunitReport: "" jsonFile: "" ignoreNoqa: false - noCache: false concurrency: 4 regoTrace: false # skip: maps document path (relative to model source, or absolute) to rules to skip. diff --git a/lint/config.go b/lint/config.go index 370f160..0c5e672 100644 --- a/lint/config.go +++ b/lint/config.go @@ -69,7 +69,6 @@ type ConfigLintSpec struct { XunitReport string `yaml:"xunitReport"` JSONFile string `yaml:"jsonFile"` IgnoreNoqa *bool `yaml:"ignoreNoqa"` - NoCache *bool `yaml:"noCache"` Concurrency *int `yaml:"concurrency"` RegoTrace *bool `yaml:"regoTrace"` Skip map[string][]ConfigSkipRule `yaml:"skip"` @@ -327,9 +326,6 @@ func mergeConfig(base *Config, overlay *Config) { if overlay.Lint.IgnoreNoqa != nil { base.Lint.IgnoreNoqa = overlay.Lint.IgnoreNoqa } - if overlay.Lint.NoCache != nil { - base.Lint.NoCache = overlay.Lint.NoCache - } if overlay.Lint.Concurrency != nil { base.Lint.Concurrency = overlay.Lint.Concurrency } diff --git a/main.go b/main.go index 4a10b2c..ecb1c19 100644 --- a/main.go +++ b/main.go @@ -327,9 +327,6 @@ func effectiveLintUseCache(config *lint.Config) bool { if config == nil { return true } - if boolValue(config.Lint.NoCache, false) { - return false - } return boolValue(config.Cache.Enable, true) } diff --git a/main_test.go b/main_test.go index 0f2d0d2..c979ce3 100644 --- a/main_test.go +++ b/main_test.go @@ -8,7 +8,6 @@ import ( func TestEffectiveLintUseCache(t *testing.T) { falseValue := false - trueValue := true tests := []struct { name string @@ -26,9 +25,6 @@ func TestEffectiveLintUseCache(t *testing.T) { Cache: lint.ConfigCacheSpec{ Enable: nil, }, - Lint: lint.ConfigLintSpec{ - NoCache: nil, - }, }, expected: true, }, @@ -41,18 +37,6 @@ func TestEffectiveLintUseCache(t *testing.T) { }, expected: false, }, - { - name: "cache disabled by lint.noCache even when cache.enable true", - config: &lint.Config{ - Cache: lint.ConfigCacheSpec{ - Enable: &trueValue, - }, - Lint: lint.ConfigLintSpec{ - NoCache: &trueValue, - }, - }, - expected: false, - }, } for _, tt := range tests { diff --git a/serve/serve.go b/serve/serve.go index 89e7700..c979556 100644 --- a/serve/serve.go +++ b/serve/serve.go @@ -355,9 +355,6 @@ func effectiveLintUseCacheForServe(config *lint.Config) bool { if config == nil { return true } - if boolValue(config.Lint.NoCache, false) { - return false - } return boolValue(config.Cache.Enable, true) } diff --git a/serve/serve_config_test.go b/serve/serve_config_test.go index 33e2865..dfbd9f0 100644 --- a/serve/serve_config_test.go +++ b/serve/serve_config_test.go @@ -8,7 +8,6 @@ import ( func TestEffectiveLintUseCacheForServe(t *testing.T) { falseValue := false - trueValue := true tests := []struct { name string @@ -21,22 +20,19 @@ func TestEffectiveLintUseCacheForServe(t *testing.T) { expected: true, }, { - name: "cache disabled by cache.enable", + name: "cache enabled by default", config: &lint.Config{ Cache: lint.ConfigCacheSpec{ - Enable: &falseValue, + Enable: nil, }, }, - expected: false, + expected: true, }, { - name: "cache disabled by lint.noCache", + name: "cache disabled by cache.enable", config: &lint.Config{ Cache: lint.ConfigCacheSpec{ - Enable: &trueValue, - }, - Lint: lint.ConfigLintSpec{ - NoCache: &trueValue, + Enable: &falseValue, }, }, expected: false, From e283b902f410caf3bd0be065510b72297b08b7fe Mon Sep 17 00:00:00 2001 From: Xiwen Cheng Date: Tue, 7 Jul 2026 23:40:09 +0200 Subject: [PATCH 3/5] improve performance by skipping the whole directory of .mendix-cache --- mpr/export_stream.go | 4 ++-- mpr/mx_units_v1.go | 5 ++--- mpr/mx_units_v2.go | 5 ++--- mpr/utils.go | 14 +++++++++++++ mpr/walk_test.go | 48 ++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 68 insertions(+), 8 deletions(-) create mode 100644 mpr/walk_test.go diff --git a/mpr/export_stream.go b/mpr/export_stream.go index 7370fad..1c5806f 100644 --- a/mpr/export_stream.go +++ b/mpr/export_stream.go @@ -146,8 +146,8 @@ func buildExportPlanV2(inputDirectory string, mprPath string) (*exportPlan, erro if walkErr != nil { return walkErr } - if strings.Contains(path, ".mendix-cache") { - return nil + if err := skipMendixCacheDir(path, info); err != nil { + return err } if info.IsDir() || !strings.HasSuffix(info.Name(), ".mxunit") { return nil diff --git a/mpr/mx_units_v1.go b/mpr/mx_units_v1.go index a3a21ac..8083f85 100644 --- a/mpr/mx_units_v1.go +++ b/mpr/mx_units_v1.go @@ -31,9 +31,8 @@ func getMprPath(inputDirectory string) (string, error) { if err != nil { return err } - if strings.Contains(path, ".mendix-cache") { - log.Debugf("Skipping system managed file %s", path) - return nil + if err := skipMendixCacheDir(path, info); err != nil { + return err } if !info.IsDir() && strings.HasSuffix(info.Name(), ".mpr") { mprPath = path diff --git a/mpr/mx_units_v2.go b/mpr/mx_units_v2.go index 39dd8d7..6e68b29 100644 --- a/mpr/mx_units_v2.go +++ b/mpr/mx_units_v2.go @@ -40,9 +40,8 @@ func readMxUnitsV2(inputDirectory string) ([]MxUnit, error) { return fmt.Errorf("error walking path %s: %v", path, err) } - if strings.Contains(path, ".mendix-cache") { - log.Debugf("Skipping system managed file %s", path) - return nil + if err := skipMendixCacheDir(path, info); err != nil { + return err } if !info.IsDir() && strings.HasSuffix(info.Name(), ".mxunit") { diff --git a/mpr/utils.go b/mpr/utils.go index 78aa5f6..11a84fa 100644 --- a/mpr/utils.go +++ b/mpr/utils.go @@ -3,6 +3,8 @@ package mpr import ( "encoding/base64" "fmt" + "os" + "path/filepath" "github.com/sirupsen/logrus" "go.mongodb.org/mongo-driver/bson" @@ -15,6 +17,18 @@ func SetLogger(logger *logrus.Logger) { log = logger } +const mendixCacheDirName = ".mendix-cache" + +// skipMendixCacheDir returns filepath.SkipDir for .mendix-cache so walks never +// descend into Mendix-managed cache content. +func skipMendixCacheDir(path string, info os.FileInfo) error { + if info.IsDir() && info.Name() == mendixCacheDirName { + log.Debugf("Skipping system managed directory %s", path) + return filepath.SkipDir + } + return nil +} + var ignoredAttributes = []string{"ID", "$ID", "Flows", "OriginPointer", "Type", "LineType", "DestinationPointer", "Image", "ImageData", "GUID", "StableId", "Size", "RelativeMiddlePoint", "Location", "OriginBezierVector", "DestinationBezierVector", "OriginConnectionIndex", "DestinationConnectionIndex"} func Contains(slice []string, str string) bool { diff --git a/mpr/walk_test.go b/mpr/walk_test.go new file mode 100644 index 0000000..d75ae8a --- /dev/null +++ b/mpr/walk_test.go @@ -0,0 +1,48 @@ +package mpr + +import ( + "os" + "path/filepath" + "testing" +) + +func TestSkipMendixCacheDir(t *testing.T) { + t.Parallel() + + root := t.TempDir() + cacheDir := filepath.Join(root, mendixCacheDirName) + if err := os.MkdirAll(filepath.Join(cacheDir, "extensions-cache", "nested"), 0755); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := os.WriteFile(filepath.Join(cacheDir, "extensions-cache", "nested", "file.dll"), []byte("x"), 0644); err != nil { + t.Fatalf("write file: %v", err) + } + + var visited []string + err := filepath.Walk(root, func(path string, info os.FileInfo, walkErr error) error { + if walkErr != nil { + return walkErr + } + if err := skipMendixCacheDir(path, info); err != nil { + return err + } + visited = append(visited, path) + return nil + }) + if err != nil { + t.Fatalf("walk: %v", err) + } + + for _, path := range visited { + rel, err := filepath.Rel(root, path) + if err != nil { + t.Fatalf("rel: %v", err) + } + if rel == mendixCacheDirName { + continue + } + if len(rel) > len(mendixCacheDirName) && rel[:len(mendixCacheDirName)+1] == mendixCacheDirName+string(os.PathSeparator) { + t.Fatalf("walk descended into mendix cache: %s", path) + } + } +} From 72c71d430fc06171ea871899d7b4f41b61374338 Mon Sep 17 00:00:00 2001 From: Xiwen Cheng Date: Wed, 8 Jul 2026 00:01:13 +0200 Subject: [PATCH 4/5] Speed up export for mpr v2 --- default.yaml | 1 + lint/config.go | 10 +- main.go | 10 + mpr/export_manifest.go | 132 ++++++++++ mpr/export_options.go | 48 ++++ mpr/export_stream.go | 498 +++++++++++++++++++++++++++++--------- mpr/export_stream_test.go | 115 +++++++++ mpr/mpr.go | 93 +++---- mpr/mpr_test.go | 12 +- mpr/mpr_v1_test.go | 6 +- mpr/mpr_v2_test.go | 6 +- mpr/mxunit_path.go | 62 +++++ mpr/mxunit_path_test.go | 105 ++++++++ serve/serve.go | 2 + 14 files changed, 925 insertions(+), 175 deletions(-) create mode 100644 mpr/export_manifest.go create mode 100644 mpr/export_options.go create mode 100644 mpr/export_stream_test.go create mode 100644 mpr/mxunit_path.go create mode 100644 mpr/mxunit_path_test.go diff --git a/default.yaml b/default.yaml index c7c82cb..47c9e6c 100644 --- a/default.yaml +++ b/default.yaml @@ -20,6 +20,7 @@ export: filter: ".*" raw: false appstore: false + concurrency: 4 serve: port: 8082 debounce: 500 diff --git a/lint/config.go b/lint/config.go index 0c5e672..582c668 100644 --- a/lint/config.go +++ b/lint/config.go @@ -60,9 +60,10 @@ func (c *ConfigRulesSpec) UnmarshalYAML(value *yaml.Node) error { } type ConfigExportSpec struct { - Filter string `yaml:"filter"` - Raw *bool `yaml:"raw"` - Appstore *bool `yaml:"appstore"` + Filter string `yaml:"filter"` + Raw *bool `yaml:"raw"` + Appstore *bool `yaml:"appstore"` + Concurrency *int `yaml:"concurrency"` } type ConfigLintSpec struct { @@ -310,6 +311,9 @@ func mergeConfig(base *Config, overlay *Config) { if overlay.Export.Appstore != nil { base.Export.Appstore = overlay.Export.Appstore } + if overlay.Export.Concurrency != nil { + base.Export.Concurrency = overlay.Export.Concurrency + } if strings.TrimSpace(overlay.Modelsource) != "" { base.Modelsource = strings.TrimSpace(overlay.Modelsource) diff --git a/main.go b/main.go index ecb1c19..815969d 100644 --- a/main.go +++ b/main.go @@ -64,6 +64,7 @@ func main() { mpr.SetLogger(log) lint.SetConfig(config) configureCache(config, projectDir) + configureExport(config) inputDirectory := config.ProjectDirectory outputDirectory := config.Modelsource @@ -298,6 +299,14 @@ func main() { } } +func configureExport(config *lint.Config) { + if config == nil { + mpr.ConfigureExportConcurrency(nil) + return + } + mpr.ConfigureExportConcurrency(config.Export.Concurrency) +} + func configureCache(config *lint.Config, projectDir string) { if config == nil { return @@ -314,6 +323,7 @@ func configureCache(config *lint.Config, projectDir string) { lint.SetCacheDirectory(filepath.Join(cacheBase, "lint")) mpr.SetPersistentYAMLCacheDirectory(filepath.Join(cacheBase, "mpr-v2-yaml")) mpr.SetPersistentYAMLCacheEnabled(boolValue(config.Cache.Enable, true)) + mpr.SetExportManifestPath(filepath.Join(cacheBase, "export-manifest.json")) } func boolValue(value *bool, fallback bool) bool { diff --git a/mpr/export_manifest.go b/mpr/export_manifest.go new file mode 100644 index 0000000..5645056 --- /dev/null +++ b/mpr/export_manifest.go @@ -0,0 +1,132 @@ +package mpr + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "sync" +) + +const exportManifestVersion = 1 + +type exportManifestEntry struct { + Name string `json:"name"` + Type string `json:"type"` + FolderPath string `json:"folderPath"` + RelativePath string `json:"relativePath"` + ContentsHash string `json:"contentsHash"` + ModTimeNs int64 `json:"modTimeNs,omitempty"` + FileSize int64 `json:"fileSize,omitempty"` +} + +type exportManifest struct { + Version int `json:"version"` + Entries map[string]exportManifestEntry `json:"entries"` +} + +var exportManifestSettings = struct { + mu sync.RWMutex + path string +}{ + path: "", +} + +func SetExportManifestPath(path string) { + exportManifestSettings.mu.Lock() + defer exportManifestSettings.mu.Unlock() + exportManifestSettings.path = strings.TrimSpace(path) +} + +func getExportManifestPath() string { + exportManifestSettings.mu.RLock() + defer exportManifestSettings.mu.RUnlock() + return exportManifestSettings.path +} + +func newExportManifest() *exportManifest { + return &exportManifest{ + Version: exportManifestVersion, + Entries: make(map[string]exportManifestEntry), + } +} + +func loadExportManifest(path string) (*exportManifest, error) { + if path == "" { + return newExportManifest(), nil + } + data, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return newExportManifest(), nil + } + return nil, fmt.Errorf("error reading export manifest: %w", err) + } + manifest := &exportManifest{} + if err := json.Unmarshal(data, manifest); err != nil { + return nil, fmt.Errorf("error parsing export manifest: %w", err) + } + if manifest.Entries == nil { + manifest.Entries = make(map[string]exportManifestEntry) + } + return manifest, nil +} + +func saveExportManifest(path string, manifest *exportManifest) error { + if path == "" || manifest == nil { + return nil + } + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + return fmt.Errorf("error creating export manifest directory: %w", err) + } + data, err := json.MarshalIndent(manifest, "", " ") + if err != nil { + return fmt.Errorf("error marshaling export manifest: %w", err) + } + tmpPath := path + ".tmp" + if err := os.WriteFile(tmpPath, data, 0644); err != nil { + return fmt.Errorf("error writing export manifest: %w", err) + } + if err := os.Rename(tmpPath, path); err != nil { + return fmt.Errorf("error saving export manifest: %w", err) + } + return nil +} + +func (m *exportManifest) entryFor(unitID, contentsHash string) (exportManifestEntry, bool) { + if m == nil { + return exportManifestEntry{}, false + } + entry, ok := m.Entries[unitID] + if !ok || entry.ContentsHash != contentsHash { + return exportManifestEntry{}, false + } + return entry, true +} + +func mxunitFileStat(path string) (modTimeNs, size int64, err error) { + info, err := os.Stat(path) + if err != nil { + return 0, 0, err + } + return info.ModTime().UnixNano(), info.Size(), nil +} + +// manifestFastPathHint returns true when mtime and size match the manifest entry. +func manifestFastPathHint(entry exportManifestEntry, mxunitPath string) bool { + if entry.ModTimeNs == 0 && entry.FileSize == 0 { + return true + } + modTimeNs, size, err := mxunitFileStat(mxunitPath) + if err != nil { + return false + } + if entry.FileSize != 0 && entry.FileSize != size { + return false + } + if entry.ModTimeNs != 0 && entry.ModTimeNs != modTimeNs { + return false + } + return true +} diff --git a/mpr/export_options.go b/mpr/export_options.go new file mode 100644 index 0000000..9c52eef --- /dev/null +++ b/mpr/export_options.go @@ -0,0 +1,48 @@ +package mpr + +import "runtime" + +const defaultMaxExportConcurrency = 4 + +var configuredExportConcurrency int + +func SetExportConcurrency(concurrency int) { + if concurrency < 0 { + concurrency = 0 + } + configuredExportConcurrency = concurrency +} + +// ConfigureExportConcurrency applies export.concurrency from config. Zero or nil uses GOMAXPROCS-based default. +func ConfigureExportConcurrency(concurrency *int) { + if concurrency == nil || *concurrency <= 0 { + SetExportConcurrency(0) + return + } + SetExportConcurrency(*concurrency) +} + +func effectiveExportConcurrency(documentCount int) int { + if documentCount <= 0 { + return 1 + } + + if configuredExportConcurrency > 0 { + if configuredExportConcurrency > documentCount { + return documentCount + } + return configuredExportConcurrency + } + + auto := runtime.GOMAXPROCS(0) + if auto < 1 { + auto = 1 + } + if auto > defaultMaxExportConcurrency { + auto = defaultMaxExportConcurrency + } + if auto > documentCount { + auto = documentCount + } + return auto +} diff --git a/mpr/export_stream.go b/mpr/export_stream.go index 1c5806f..a1fe2fb 100644 --- a/mpr/export_stream.go +++ b/mpr/export_stream.go @@ -1,13 +1,15 @@ package mpr import ( + "crypto/sha256" "database/sql" - "encoding/base64" + "encoding/hex" "fmt" "os" "path/filepath" "regexp" - "strings" + "sync" + "sync/atomic" _ "github.com/glebarez/go-sqlite" "go.mongodb.org/mongo-driver/bson" @@ -23,25 +25,69 @@ var documentContainmentTypes = map[string]struct{}{ } type exportDocumentDescriptor struct { - UnitID string - Name string - Type string - ContainerID string - Path string + UnitID string + Name string + Type string + ContainerID string + Path string + ContentsHash string +} + +type cachedUnitContent struct { + Contents bson.M + ContentsHash string } type exportPlan struct { - Modules []MxModule - Documents []exportDocumentDescriptor - Load func(unitID string) (bson.M, error) - Close func() error + Modules []MxModule + Documents []exportDocumentDescriptor + unitCache map[string]cachedUnitContent + unitCacheMu sync.Mutex + mxunitPaths map[string]string + manifest *exportManifest + manifestPath string + manifestMu sync.Mutex + Close func() error } -func buildExportPlan(inputDirectory string) (*exportPlan, error) { - mprPath, err := getMprPath(inputDirectory) +func (p *exportPlan) loadDocument(unitID string) (bson.M, error) { + p.unitCacheMu.Lock() + if cached, ok := p.unitCache[unitID]; ok { + p.unitCacheMu.Unlock() + return cached.Contents, nil + } + p.unitCacheMu.Unlock() + + mxunitPath, ok := p.mxunitPaths[unitID] + if !ok { + return nil, fmt.Errorf("mxunit path not found for unit %s", unitID) + } + + _, result, hash, err := readMxUnitAtPath(mxunitPath) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to read mxunit %s: %w", mxunitPath, err) + } + + p.unitCacheMu.Lock() + p.unitCache[unitID] = cachedUnitContent{Contents: result, ContentsHash: hash} + p.unitCacheMu.Unlock() + return result, nil +} + +func readMxUnitAtPath(path string) ([]byte, bson.M, string, error) { + contents, err := os.ReadFile(path) + if err != nil { + return nil, nil, "", err } + var result bson.M + if err := bson.Unmarshal(contents, &result); err != nil { + return nil, nil, "", fmt.Errorf("unable to unmarshal BSON content for %s: %w", path, err) + } + sum := sha256.Sum256(contents) + return contents, result, hex.EncodeToString(sum[:]), nil +} + +func buildExportPlan(inputDirectory string, mprPath string) (*exportPlan, error) { mprVersion, err := getMprVersion(mprPath) if err != nil { return nil, fmt.Errorf("error getting mpr version: %v", err) @@ -68,6 +114,7 @@ func buildExportPlanV1(mprPath string) (*exportPlan, error) { modules := make([]MxModule, 0) folders := make([]MxFolder, 0) documents := make([]exportDocumentDescriptor, 0) + unitCache := make(map[string]cachedUnitContent) for rows.Next() { var containmentName string @@ -83,9 +130,16 @@ func buildExportPlanV1(mprPath string) (*exportPlan, error) { return nil, fmt.Errorf("error parsing unit: %v", err) } + encodedUnitID := encodeUnitID(unitID) + contentHash := sha256.Sum256(contents) + unitCache[encodedUnitID] = cachedUnitContent{ + Contents: result, + ContentsHash: hex.EncodeToString(contentHash[:]), + } + unit := MxUnit{ - UnitID: base64.StdEncoding.EncodeToString(unitID), - ContainerID: base64.StdEncoding.EncodeToString(containerID), + UnitID: encodedUnitID, + ContainerID: encodeContainerID(containerID), ContainmentName: containmentName, } appendUnitDescriptor(unit, result, &modules, &folders, &documents) @@ -98,112 +152,125 @@ func buildExportPlanV1(mprPath string) (*exportPlan, error) { connectFolderParents(folders) for i := range documents { documents[i].Path = getMxDocumentPath(documents[i].ContainerID, folders) - } - - loadDocument := func(unitID string) (bson.M, error) { - rawUnitID, err := base64.StdEncoding.DecodeString(unitID) - if err != nil { - return nil, fmt.Errorf("failed decoding unit id %s: %w", unitID, err) - } - - var contents []byte - if err := db.QueryRow("SELECT Contents FROM Unit WHERE UnitID = ?", rawUnitID).Scan(&contents); err != nil { - return nil, fmt.Errorf("failed to query contents for unit %s: %w", unitID, err) - } - var result bson.M - if err := bson.Unmarshal(contents, &result); err != nil { - return nil, fmt.Errorf("failed to parse contents for unit %s: %w", unitID, err) + if cached, ok := unitCache[documents[i].UnitID]; ok { + documents[i].ContentsHash = cached.ContentsHash } - return result, nil } + _ = db.Close() return &exportPlan{ Modules: modules, Documents: documents, - Load: loadDocument, - Close: db.Close, + unitCache: unitCache, + Close: func() error { + return nil + }, }, nil } func buildExportPlanV2(inputDirectory string, mprPath string) (*exportPlan, error) { - units, err := getMxUnitsV2(mprPath) + manifestPath := getExportManifestPath() + manifest, err := loadExportManifest(manifestPath) if err != nil { return nil, err } - unitHeaders := make(map[string]MxUnit, len(units)) - for _, unit := range units { - unitHeaders[unit.UnitID] = unit + db, err := sql.Open("sqlite", mprPath) + if err != nil { + return nil, fmt.Errorf("error opening database: %v", err) + } + defer db.Close() + + rows, err := db.Query("SELECT UnitID, ContainerID, ContainmentName, ContentsHash FROM Unit") + if err != nil { + return nil, fmt.Errorf("error querying units: %v", err) } + defer rows.Close() - mxUnitPaths := make(map[string]string, len(units)) + unitCache := make(map[string]cachedUnitContent) + mxunitPaths := make(map[string]string, 256) modules := make([]MxModule, 0) folders := make([]MxFolder, 0) documents := make([]exportDocumentDescriptor, 0) - mprContentsDirectory := filepath.Join(inputDirectory, "mprcontents") - err = filepath.Walk(mprContentsDirectory, func(path string, info os.FileInfo, walkErr error) error { - if walkErr != nil { - return walkErr - } - if err := skipMendixCacheDir(path, info); err != nil { - return err + for rows.Next() { + var containmentName string + var dbContentsHash sql.NullString + var unitID, containerID []byte + if err := rows.Scan(&unitID, &containerID, &containmentName, &dbContentsHash); err != nil { + return nil, fmt.Errorf("error scanning unit: %v", err) } - if info.IsDir() || !strings.HasSuffix(info.Name(), ".mxunit") { - return nil + + encodedUnitID := encodeUnitID(unitID) + contentsHashHex := "" + if dbContentsHash.Valid { + contentsHashHex, err = contentsHashHexFromDB(dbContentsHash.String) + if err != nil { + return nil, fmt.Errorf("error decoding ContentsHash for unit %s: %w", encodedUnitID, err) + } } - contents, err := os.ReadFile(path) + mxunitPath, err := mxunitPathForUnitID(inputDirectory, unitID) if err != nil { - return fmt.Errorf("error reading file %s: %v", path, err) + return nil, fmt.Errorf("error resolving mxunit path for unit %s: %w", encodedUnitID, err) } - var result bson.M - if err := bson.Unmarshal(contents, &result); err != nil { - return fmt.Errorf("unable to unmarshal BSON content for %s: %w", path, err) + mxunitPaths[encodedUnitID] = mxunitPath + + unit := MxUnit{ + UnitID: encodedUnitID, + ContainerID: encodeContainerID(containerID), + ContainmentName: containmentName, } - unitID, err := extractUnitIDFromRawContents(result, path) - if err != nil { - return err + if isStructureContainment(containmentName) { + _, result, fileHash, err := readMxUnitAtPath(mxunitPath) + if err != nil { + return nil, fmt.Errorf("error reading structure unit %s: %w", mxunitPath, err) + } + hash := contentsHashHex + if hash == "" { + hash = fileHash + } + unitCache[encodedUnitID] = cachedUnitContent{Contents: result, ContentsHash: hash} + appendUnitDescriptor(unit, result, &modules, &folders, &documents) + continue } - header, exists := unitHeaders[unitID] - if !exists { - return fmt.Errorf("unable to find unit with ID: %s", unitID) + + if _, ok := documentContainmentTypes[containmentName]; ok { + doc := exportDocumentDescriptor{ + UnitID: encodedUnitID, + ContainerID: unit.ContainerID, + ContentsHash: contentsHashHex, + } + if entry, ok := manifest.entryFor(encodedUnitID, contentsHashHex); ok { + doc.Name = entry.Name + doc.Type = entry.Type + doc.Path = entry.FolderPath + } + documents = append(documents, doc) } - mxUnitPaths[unitID] = path - appendUnitDescriptor(header, result, &modules, &folders, &documents) - return nil - }) - if err != nil { - return nil, fmt.Errorf("error processing mprcontents: %v", err) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("error iterating units: %v", err) } connectFolderParents(folders) for i := range documents { - documents[i].Path = getMxDocumentPath(documents[i].ContainerID, folders) - } - - loadDocument := func(unitID string) (bson.M, error) { - mxunitPath, ok := mxUnitPaths[unitID] - if !ok { - return nil, fmt.Errorf("mxunit path not found for unit %s", unitID) - } - - contents, err := os.ReadFile(mxunitPath) - if err != nil { - return nil, fmt.Errorf("failed to read mxunit %s: %w", mxunitPath, err) - } - var result bson.M - if err := bson.Unmarshal(contents, &result); err != nil { - return nil, fmt.Errorf("failed to unmarshal mxunit %s: %w", mxunitPath, err) + if documents[i].Path == "" { + documents[i].Path = getMxDocumentPath(documents[i].ContainerID, folders) } - return result, nil } + log.Debugf("Built export plan from SQLite: %d modules, %d documents (%d structure units cached)", + len(modules), len(documents), len(unitCache)) + return &exportPlan{ - Modules: modules, - Documents: documents, - Load: loadDocument, + Modules: modules, + Documents: documents, + unitCache: unitCache, + mxunitPaths: mxunitPaths, + manifest: manifest, + manifestPath: manifestPath, Close: func() error { return nil }, @@ -265,6 +332,33 @@ func connectFolderParents(folders []MxFolder) { } } +type exportDirCache struct { + mu sync.Mutex + dirs map[string]struct{} +} + +func newExportDirCache() *exportDirCache { + return &exportDirCache{dirs: make(map[string]struct{})} +} + +func (c *exportDirCache) mkdir(path string) error { + c.mu.Lock() + if _, ok := c.dirs[path]; ok { + c.mu.Unlock() + return nil + } + c.mu.Unlock() + + if err := os.MkdirAll(path, 0755); err != nil { + return fmt.Errorf("error creating directory: %v", err) + } + + c.mu.Lock() + c.dirs[path] = struct{}{} + c.mu.Unlock() + return nil +} + func exportDocumentsFromPlan(plan *exportPlan, outputDirectory string, raw bool, filter string) (int, error) { var err error var filterRegex *regexp.Regexp @@ -276,37 +370,207 @@ func exportDocumentsFromPlan(plan *exportPlan, outputDirectory string, raw bool, log.Infof("Applying filter: %s", filter) } - exportedCount := 0 + if len(plan.Documents) == 0 { + log.Infof("Found 0 documents") + return 0, nil + } + + concurrency := effectiveExportConcurrency(len(plan.Documents)) + dirCache := newExportDirCache() + jobCh := make(chan exportDocumentDescriptor) + var wg sync.WaitGroup + var exportedCount atomic.Int64 + var exportErr atomic.Value + + for worker := 0; worker < concurrency; worker++ { + wg.Add(1) + go func() { + defer wg.Done() + for document := range jobCh { + if exportErr.Load() != nil { + return + } + exported, err := exportDocument(plan, document, outputDirectory, raw, filterRegex, dirCache) + if err != nil { + exportErr.Store(err) + return + } + if exported { + exportedCount.Add(1) + } + } + }() + } + for _, document := range plan.Documents { - if filterRegex != nil && !filterRegex.MatchString(document.Name) { - log.Debugf("Skipping document '%s' (does not match filter)", document.Name) - continue + jobCh <- document + } + close(jobCh) + wg.Wait() + + if stored := exportErr.Load(); stored != nil { + return 0, stored.(error) + } + + if err := saveExportManifest(plan.manifestPath, plan.manifest); err != nil { + log.Warnf("Could not save export manifest: %v", err) + } + + count := int(exportedCount.Load()) + if filterRegex != nil { + log.Infof("Exported %d documents matching filter (out of %d total)", count, len(plan.Documents)) + } else { + log.Infof("Found %d documents", count) + } + return count, nil +} + +func (p *exportPlan) recordManifestEntry(unitID string, entry exportManifestEntry) { + p.manifestMu.Lock() + defer p.manifestMu.Unlock() + if p.manifest == nil { + p.manifest = newExportManifest() + } + p.manifest.Entries[unitID] = entry +} + +func (p *exportPlan) tryFastSkipExport(document exportDocumentDescriptor, outputDirectory string, raw bool) (bool, error) { + entry, ok := p.manifest.entryFor(document.UnitID, document.ContentsHash) + if !ok || entry.RelativePath == "" { + return false, nil + } + if mxunitPath, exists := p.mxunitPaths[document.UnitID]; exists { + if !manifestFastPathHint(entry, mxunitPath) { + log.Debugf("Manifest mtime/size hint mismatch for %s, verifying via hash", document.Name) } + } - attributes, err := plan.Load(document.UnitID) - if err != nil { - return 0, fmt.Errorf("error loading document %s: %w", document.Name, err) + cachedYAML, found, err := readYAMLFromPersistentCache(document.ContentsHash, raw) + if err != nil { + return false, err + } + if !found { + return false, nil + } + + outPath := filepath.Join(outputDirectory, entry.RelativePath) + return outputFileMatches(outPath, cachedYAML) +} + +func (p *exportPlan) tryExportFromYAMLCache(document exportDocumentDescriptor, outputDirectory string, raw bool, dirCache *exportDirCache) (string, bool, error) { + entry, ok := p.manifest.entryFor(document.UnitID, document.ContentsHash) + if !ok || entry.RelativePath == "" { + return "", false, nil + } + + cachedYAML, found, err := readYAMLFromPersistentCache(document.ContentsHash, raw) + if err != nil { + return "", false, err + } + if !found { + return "", false, nil + } + + outPath := filepath.Join(outputDirectory, entry.RelativePath) + if err := dirCache.mkdir(filepath.Dir(outPath)); err != nil { + return "", false, err + } + if same, err := outputFileMatches(outPath, cachedYAML); err != nil { + return "", false, err + } else if same { + return entry.RelativePath, true, nil + } + if err := os.WriteFile(outPath, cachedYAML, 0644); err != nil { + return "", false, fmt.Errorf("error writing cached export file: %w", err) + } + return entry.RelativePath, true, nil +} + +func exportDocument(plan *exportPlan, document exportDocumentDescriptor, outputDirectory string, raw bool, filterRegex *regexp.Regexp, dirCache *exportDirCache) (bool, error) { + doc := document + + if doc.Name == "" || doc.Type == "" { + if entry, ok := plan.manifest.entryFor(doc.UnitID, doc.ContentsHash); ok { + doc.Name = entry.Name + doc.Type = entry.Type + if doc.Path == "" { + doc.Path = entry.FolderPath + } } + } - if docType, _ := attributes["$Type"].(string); docType == microflowDocumentType { - addMicroflowPseudocode(document.Name, attributes) + var attributes bson.M + if doc.Name == "" || doc.Type == "" { + var err error + attributes, err = plan.loadDocument(doc.UnitID) + if err != nil { + return false, fmt.Errorf("error loading document metadata %s: %w", doc.UnitID, err) + } + if doc.Name == "" { + doc.Name, _ = attributes["Name"].(string) } + if doc.Type == "" { + doc.Type, _ = attributes["$Type"].(string) + } + } - if err := writeDocumentToDisk(document, outputDirectory, cleanData(attributes, raw)); err != nil { - return 0, err + if filterRegex != nil && !filterRegex.MatchString(doc.Name) { + log.Debugf("Skipping document '%s' (does not match filter)", doc.Name) + return false, nil + } + + if skipped, err := plan.tryFastSkipExport(doc, outputDirectory, raw); err != nil { + return false, err + } else if skipped { + log.Debugf("Skipping unchanged document '%s'", doc.Name) + return true, nil + } + + if relPath, ok, err := plan.tryExportFromYAMLCache(doc, outputDirectory, raw, dirCache); err != nil { + return false, err + } else if ok { + plan.recordManifestEntry(doc.UnitID, manifestEntryForDocument(doc, relPath, plan.mxunitPaths[doc.UnitID])) + return true, nil + } + + if attributes == nil { + var err error + attributes, err = plan.loadDocument(doc.UnitID) + if err != nil { + return false, fmt.Errorf("error loading document %s: %w", doc.Name, err) } - exportedCount++ } - if filterRegex != nil { - log.Infof("Exported %d documents matching filter (out of %d total)", exportedCount, len(plan.Documents)) - } else { - log.Infof("Found %d documents", len(plan.Documents)) + if docType, _ := attributes["$Type"].(string); docType == microflowDocumentType { + addMicroflowPseudocode(doc.Name, attributes) } - return exportedCount, nil + + relPath, err := writeDocumentToDisk(doc, outputDirectory, cleanData(attributes, raw), raw, dirCache) + if err != nil { + return false, err + } + plan.recordManifestEntry(doc.UnitID, manifestEntryForDocument(doc, relPath, plan.mxunitPaths[doc.UnitID])) + return true, nil } -func writeDocumentToDisk(document exportDocumentDescriptor, outputDirectory string, attributes map[string]interface{}) error { +func manifestEntryForDocument(document exportDocumentDescriptor, relativePath, mxunitPath string) exportManifestEntry { + entry := exportManifestEntry{ + Name: document.Name, + Type: document.Type, + FolderPath: document.Path, + RelativePath: relativePath, + ContentsHash: document.ContentsHash, + } + if mxunitPath != "" { + if modTimeNs, size, err := mxunitFileStat(mxunitPath); err == nil { + entry.ModTimeNs = modTimeNs + entry.FileSize = size + } + } + return entry +} + +func writeDocumentToDisk(document exportDocumentDescriptor, outputDirectory string, attributes map[string]interface{}, raw bool, dirCache *exportDirCache) (string, error) { sanitizedPath := sanitizePath(document.Path) if sanitizedPath != document.Path { log.Warnf("Sanitized path: '%s' -> '%s'", document.Path, sanitizedPath) @@ -325,21 +589,25 @@ func writeDocumentToDisk(document exportDocumentDescriptor, outputDirectory stri adjustedPath, adjustedFilename, err := validatePathLength(outputDirectory, sanitizedPath, fname) if err != nil { - return fmt.Errorf("error adjusting path length: %v", err) + return "", fmt.Errorf("error adjusting path length: %v", err) } directory := filepath.Join(outputDirectory, adjustedPath) - if _, err := os.Stat(directory); os.IsNotExist(err) { - if err := os.MkdirAll(directory, 0755); err != nil { - return fmt.Errorf("error creating directory: %v", err) - } + if err := dirCache.mkdir(directory); err != nil { + return "", err } - if err := writeFile(filepath.Join(directory, adjustedFilename), attributes); err != nil { + outPath := filepath.Join(directory, adjustedFilename) + if err := writeFileWithPersistentCache(outPath, attributes, document.ContentsHash, raw); err != nil { log.Errorf("Error writing file: %v", err) - return err + return "", err } - return nil + + relPath, err := filepath.Rel(outputDirectory, outPath) + if err != nil { + return filepath.Join(adjustedPath, adjustedFilename), nil + } + return relPath, nil } func extractUnitIDFromRawContents(data map[string]interface{}, path string) (string, error) { @@ -351,7 +619,7 @@ func extractUnitIDFromRawContents(data map[string]interface{}, path string) (str switch id := idData.(type) { case primitive.Binary: if len(id.Data) >= 16 { - return base64.StdEncoding.EncodeToString(id.Data), nil + return encodeUnitID(id.Data), nil } case map[string]interface{}: if dataStr, ok := id["Data"].(string); ok && dataStr != "" { @@ -361,7 +629,7 @@ func extractUnitIDFromRawContents(data map[string]interface{}, path string) (str switch dataBytes := dataVal.(type) { case primitive.Binary: if len(dataBytes.Data) >= 16 { - return base64.StdEncoding.EncodeToString(dataBytes.Data), nil + return encodeUnitID(dataBytes.Data), nil } case []interface{}: bytes := make([]byte, 0, len(dataBytes)) @@ -375,11 +643,11 @@ func extractUnitIDFromRawContents(data map[string]interface{}, path string) (str } } if len(bytes) >= 16 { - return base64.StdEncoding.EncodeToString(bytes), nil + return encodeUnitID(bytes), nil } case []byte: if len(dataBytes) >= 16 { - return base64.StdEncoding.EncodeToString(dataBytes), nil + return encodeUnitID(dataBytes), nil } } } diff --git a/mpr/export_stream_test.go b/mpr/export_stream_test.go new file mode 100644 index 0000000..060143a --- /dev/null +++ b/mpr/export_stream_test.go @@ -0,0 +1,115 @@ +package mpr + +import ( + "os" + "path/filepath" + "testing" +) + +func TestEffectiveExportConcurrency(t *testing.T) { + t.Parallel() + + SetExportConcurrency(0) + t.Cleanup(func() { + SetExportConcurrency(0) + }) + + if got := effectiveExportConcurrency(0); got != 1 { + t.Fatalf("expected 1 for zero documents, got %d", got) + } + + SetExportConcurrency(8) + if got := effectiveExportConcurrency(3); got != 3 { + t.Fatalf("expected concurrency capped to document count 3, got %d", got) + } +} + +func TestExportPlanLoadUsesCache(t *testing.T) { + plan := &exportPlan{ + unitCache: map[string]cachedUnitContent{ + "unit-1": { + Contents: map[string]interface{}{"Name": "Cached"}, + ContentsHash: "abc123", + }, + }, + } + + contents, err := plan.loadDocument("unit-1") + if err != nil { + t.Fatalf("loadDocument() unexpected error: %v", err) + } + if contents["Name"] != "Cached" { + t.Fatalf("expected cached contents, got %#v", contents) + } +} + +func TestOutputFileMatches(t *testing.T) { + tmpDir, err := os.MkdirTemp("", "mpr-output-match-*") + if err != nil { + t.Fatalf("mkdir temp: %v", err) + } + defer os.RemoveAll(tmpDir) + + content := []byte("name: Example\n") + path := filepath.Join(tmpDir, "example.yaml") + if err := os.WriteFile(path, content, 0644); err != nil { + t.Fatalf("write file: %v", err) + } + + same, err := outputFileMatches(path, content) + if err != nil { + t.Fatalf("outputFileMatches() error: %v", err) + } + if !same { + t.Fatal("expected matching output file to be detected") + } + + changed, err := outputFileMatches(path, []byte("name: Changed\n")) + if err != nil { + t.Fatalf("outputFileMatches() error: %v", err) + } + if changed { + t.Fatal("expected different content to not match") + } +} + +func TestWriteFileWithPersistentCacheSkipsUnchangedOutput(t *testing.T) { + tmpDir, err := os.MkdirTemp("", "mpr-output-skip-*") + if err != nil { + t.Fatalf("mkdir temp: %v", err) + } + defer os.RemoveAll(tmpDir) + + cacheDir := filepath.Join(tmpDir, "cache") + SetPersistentYAMLCacheDirectory(cacheDir) + SetPersistentYAMLCacheEnabled(true) + t.Cleanup(func() { + SetPersistentYAMLCacheDirectory("") + SetPersistentYAMLCacheEnabled(true) + }) + + contents := map[string]interface{}{"Name": "Example"} + hash := "hash-for-skip-test" + outPath := filepath.Join(tmpDir, "doc.yaml") + + if err := writeFileWithPersistentCache(outPath, contents, hash, false); err != nil { + t.Fatalf("first write failed: %v", err) + } + + before, err := os.Stat(outPath) + if err != nil { + t.Fatalf("stat output: %v", err) + } + + if err := writeFileWithPersistentCache(outPath, contents, hash, false); err != nil { + t.Fatalf("second write failed: %v", err) + } + + after, err := os.Stat(outPath) + if err != nil { + t.Fatalf("stat output after second write: %v", err) + } + if !before.ModTime().Equal(after.ModTime()) { + t.Fatal("expected unchanged output file to be left untouched") + } +} diff --git a/mpr/mpr.go b/mpr/mpr.go index 43494e7..3b09896 100644 --- a/mpr/mpr.go +++ b/mpr/mpr.go @@ -1,6 +1,7 @@ package mpr import ( + "bytes" "crypto/sha256" "database/sql" "encoding/hex" @@ -64,18 +65,12 @@ func getPersistentYAMLCacheSettings() (string, bool) { } func ExportModel(inputDirectory string, outputDirectory string, raw bool, appstore bool, filter string) error { - - // create tmp directory in user tmp directory - tmpDir := filepath.Join(os.TempDir(), "mxlint") - if err := os.MkdirAll(tmpDir, 0755); err != nil { - return fmt.Errorf("error creating tmp directory: %v", err) + if err := os.MkdirAll(outputDirectory, 0755); err != nil { + return fmt.Errorf("error creating output directory: %v", err) } - log.Debugf("Created tmp directory: %s", tmpDir) - defer os.RemoveAll(tmpDir) - log.Infof("Exporting to %s", tmpDir) + log.Infof("Exporting to %s", outputDirectory) - // Check if we can find an MPR file mprPath, err := getMprPath(inputDirectory) if err != nil { return fmt.Errorf("error finding MPR file: %v", err) @@ -84,7 +79,7 @@ func ExportModel(inputDirectory string, outputDirectory string, raw bool, appsto return fmt.Errorf("no MPR file found in directory: %s", inputDirectory) } - plan, err := buildExportPlan(inputDirectory) + plan, err := buildExportPlan(inputDirectory, mprPath) if err != nil { return fmt.Errorf("error building export plan: %v", err) } @@ -95,46 +90,24 @@ func ExportModel(inputDirectory string, outputDirectory string, raw bool, appsto }() modules := plan.Modules - if err := exportMetadata(inputDirectory, tmpDir, modules); err != nil { + if err := exportMetadata(mprPath, outputDirectory, modules); err != nil { return fmt.Errorf("error exporting metadata: %v", err) } exportedCount := 0 if filter != "^Metadata$" { - exportedCount, err = exportDocumentsFromPlan(plan, tmpDir, raw, filter) + exportedCount, err = exportDocumentsFromPlan(plan, outputDirectory, raw, filter) if err != nil { return fmt.Errorf("error exporting units: %v", err) } } - // remove output directory if it exists - if _, err := os.Stat(outputDirectory); os.IsNotExist(err) { - if err := os.MkdirAll(outputDirectory, 0755); err != nil { - return fmt.Errorf("error creating directory: %v", err) - } - } - - // Ensure both source and destination directories exist before syncing - if _, err := os.Stat(tmpDir); os.IsNotExist(err) { - return fmt.Errorf("source directory does not exist: %v", err) - } - - if _, err := os.Stat(outputDirectory); os.IsNotExist(err) { - return fmt.Errorf("destination directory does not exist: %v", err) - } - - // copy tmp directory to output directory - err = syncDirectories(tmpDir, outputDirectory) - if err != nil { - return fmt.Errorf("error moving tmp directory to output directory: %v", err) - } - if !appstore { - // remove appstore modules - removeAppstoreModules(outputDirectory, modules) + if err := removeAppstoreModules(outputDirectory, modules); err != nil { + return fmt.Errorf("error removing appstore modules: %v", err) + } } - // Generate app.yaml with file structure only if documents were exported if exportedCount > 0 { if err := generateAppYaml(outputDirectory); err != nil { return fmt.Errorf("error generating app.yaml: %v", err) @@ -176,13 +149,7 @@ func getMprVersion(MPRFilePath string) (int, error) { } } -func exportMetadata(inputDirectory string, outputDirectory string, modules []MxModule) error { - - mprPath, err := getMprPath(inputDirectory) - if err != nil { - return err - } - +func exportMetadata(mprPath string, outputDirectory string, modules []MxModule) error { mprVersion, err := getMprVersion(mprPath) if err != nil { return fmt.Errorf("error getting mpr version: %v", err) @@ -664,27 +631,47 @@ func writeFileWithPersistentCache(filepath string, contents map[string]interface if err != nil { return err } + var yamlstring []byte if found { - if err := os.WriteFile(filepath, cachedYAML, 0644); err != nil { - return fmt.Errorf("error writing cached file: %v", err) + yamlstring = cachedYAML + } else { + yamlstring, err = renderYAML(contents) + if err != nil { + return err + } + if err := writeYAMLToPersistentCache(contentsHash, raw, yamlstring); err != nil { + log.Debugf("Could not persist YAML cache for hash %s: %v", contentsHash, err) } - return nil } - yamlstring, err := renderYAML(contents) - if err != nil { + if unchanged, err := outputFileMatches(filepath, yamlstring); err != nil { return err + } else if unchanged { + return nil } if err := os.WriteFile(filepath, yamlstring, 0644); err != nil { return fmt.Errorf("error writing file: %v", err) } + return nil +} - if err := writeYAMLToPersistentCache(contentsHash, raw, yamlstring); err != nil { - log.Debugf("Could not persist YAML cache for hash %s: %v", contentsHash, err) +func outputFileMatches(path string, content []byte) (bool, error) { + info, err := os.Stat(path) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err } - - return nil + if info.Size() != int64(len(content)) { + return false, nil + } + existing, err := os.ReadFile(path) + if err != nil { + return false, err + } + return bytes.Equal(existing, content), nil } func getPersistentYAMLCacheDir() string { diff --git a/mpr/mpr_test.go b/mpr/mpr_test.go index bcce707..f3c6153 100644 --- a/mpr/mpr_test.go +++ b/mpr/mpr_test.go @@ -870,7 +870,11 @@ func TestExportMetadata(t *testing.T) { } modules := getMxModules(units) - err = exportMetadata(tt.inputDir, tt.outputDir, modules) + mprPath, err := getMprPath(tt.inputDir) + if err != nil && !tt.expectError { + t.Fatalf("Failed to resolve MPR path: %v", err) + } + err = exportMetadata(mprPath, tt.outputDir, modules) if tt.expectError && err == nil { t.Errorf("exportMetadata() expected error but got none") } @@ -1449,7 +1453,11 @@ func TestExportMetadata_SortsModulesByName(t *testing.T) { {Name: "beta", ID: "1"}, } - if err := exportMetadata("./../resources/app-mpr-v1", tmpDir, modules); err != nil { + mprPath, err := getMprPath("./../resources/app-mpr-v1") + if err != nil { + t.Fatalf("Failed to resolve MPR path: %v", err) + } + if err := exportMetadata(mprPath, tmpDir, modules); err != nil { t.Fatalf("exportMetadata() unexpected error: %v", err) } diff --git a/mpr/mpr_v1_test.go b/mpr/mpr_v1_test.go index 77522ba..caf35e2 100644 --- a/mpr/mpr_v1_test.go +++ b/mpr/mpr_v1_test.go @@ -14,7 +14,11 @@ import ( // TestAdd tests the Add function to ensure it returns correct results. func TestMPRMetadata(t *testing.T) { t.Run("single-mpr", func(t *testing.T) { - if err := exportMetadata("./../resources/app-mpr-v1", "./../tmp", nil); err != nil { + mprPath, err := getMprPath("./../resources/app-mpr-v1") + if err != nil { + t.Fatalf("Failed to resolve MPR path: %v", err) + } + if err := exportMetadata(mprPath, "./../tmp", nil); err != nil { t.Errorf("Failed to export metadata from MPR file") } diff --git a/mpr/mpr_v2_test.go b/mpr/mpr_v2_test.go index feabc47..2fd97ab 100644 --- a/mpr/mpr_v2_test.go +++ b/mpr/mpr_v2_test.go @@ -11,7 +11,11 @@ import ( // TestAdd tests the Add function to ensure it returns correct results. func TestMPRV2Metadata(t *testing.T) { t.Run("single-mpr", func(t *testing.T) { - if err := exportMetadata("./../resources/app-mpr-v2", "./../tmp", nil); err != nil { + mprPath, err := getMprPath("./../resources/app-mpr-v2") + if err != nil { + t.Fatalf("Failed to resolve MPR path: %v", err) + } + if err := exportMetadata(mprPath, "./../tmp", nil); err != nil { t.Errorf("Failed to export metadata from MPR file") } diff --git a/mpr/mxunit_path.go b/mpr/mxunit_path.go new file mode 100644 index 0000000..be79189 --- /dev/null +++ b/mpr/mxunit_path.go @@ -0,0 +1,62 @@ +package mpr + +import ( + "encoding/base64" + "encoding/hex" + "fmt" + "path/filepath" + "strings" +) + +// unitIDToMxunitGUID converts a Mendix UnitID blob to the UUID string used in mxunit paths. +// Mendix stores GUID bytes in little-endian field order (.NET Guid layout). +func unitIDToMxunitGUID(unitID []byte) (string, error) { + if len(unitID) != 16 { + return "", fmt.Errorf("expected 16-byte unit id, got %d bytes", len(unitID)) + } + b := unitID + return fmt.Sprintf( + "%08x-%04x-%04x-%04x-%012x", + uint32(b[3])<<24|uint32(b[2])<<16|uint32(b[1])<<8|uint32(b[0]), + uint16(b[5])<<8|uint16(b[4]), + uint16(b[7])<<8|uint16(b[6]), + uint16(b[8])<<8|uint16(b[9]), + uint64(b[10])<<40|uint64(b[11])<<32|uint64(b[12])<<24|uint64(b[13])<<16|uint64(b[14])<<8|uint64(b[15]), + ), nil +} + +func encodeUnitID(unitID []byte) string { + return base64.StdEncoding.EncodeToString(unitID) +} + +func encodeContainerID(containerID []byte) string { + if len(containerID) == 0 { + return "" + } + return base64.StdEncoding.EncodeToString(containerID) +} + +func mxunitPathForUnitID(inputDirectory string, unitID []byte) (string, error) { + guid, err := unitIDToMxunitGUID(unitID) + if err != nil { + return "", err + } + return filepath.Join(inputDirectory, "mprcontents", guid[0:2], guid[2:4], guid+".mxunit"), nil +} + +// contentsHashHexFromDB converts Mendix SQLite ContentsHash (base64-encoded SHA-256) to hex. +func contentsHashHexFromDB(dbHash string) (string, error) { + dbHash = strings.TrimSpace(dbHash) + if dbHash == "" { + return "", nil + } + raw, err := base64.StdEncoding.DecodeString(dbHash) + if err != nil { + return "", fmt.Errorf("invalid ContentsHash: %w", err) + } + return hex.EncodeToString(raw), nil +} + +func isStructureContainment(containmentName string) bool { + return containmentName == "Modules" || containmentName == "Folders" || containmentName == "" +} diff --git a/mpr/mxunit_path_test.go b/mpr/mxunit_path_test.go new file mode 100644 index 0000000..ac36a3e --- /dev/null +++ b/mpr/mxunit_path_test.go @@ -0,0 +1,105 @@ +package mpr + +import ( + "encoding/base64" + "encoding/hex" + "os" + "path/filepath" + "testing" +) + +func TestUnitIDToMxunitGUID(t *testing.T) { + t.Parallel() + + // UnitID bytes from App.mpr for mxunit 82d3944d-781d-454f-b408-15318fdc23b8 + unitID, err := hex.DecodeString("4d94d3821d784f45b40815318fdc23b8") + if err != nil { + t.Fatalf("decode unit id: %v", err) + } + got, err := unitIDToMxunitGUID(unitID) + if err != nil { + t.Fatalf("unitIDToMxunitGUID() error: %v", err) + } + want := "82d3944d-781d-454f-b408-15318fdc23b8" + if got != want { + t.Fatalf("expected %s, got %s", want, got) + } +} + +func TestMxunitPathForUnitID(t *testing.T) { + t.Parallel() + + root := t.TempDir() + known := "82d3944d-781d-454f-b408-15318fdc23b8" + unitID, err := hex.DecodeString("4d94d3821d784f45b40815318fdc23b8") + if err != nil { + t.Fatalf("decode unit id: %v", err) + } + targetDir := filepath.Join(root, "mprcontents", known[0:2], known[2:4]) + if err := os.MkdirAll(targetDir, 0755); err != nil { + t.Fatalf("mkdir: %v", err) + } + target := filepath.Join(targetDir, known+".mxunit") + if err := os.WriteFile(target, []byte("test"), 0644); err != nil { + t.Fatalf("write: %v", err) + } + + got, err := mxunitPathForUnitID(root, unitID) + if err != nil { + t.Fatalf("mxunitPathForUnitID() error: %v", err) + } + if got != target { + t.Fatalf("expected %s, got %s", target, got) + } +} + +func TestContentsHashHexFromDB(t *testing.T) { + t.Parallel() + + sum, err := hex.DecodeString("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855") + if err != nil { + t.Fatalf("decode: %v", err) + } + dbHash := base64.StdEncoding.EncodeToString(sum) + got, err := contentsHashHexFromDB(dbHash) + if err != nil { + t.Fatalf("contentsHashHexFromDB() error: %v", err) + } + if got != hex.EncodeToString(sum) { + t.Fatalf("unexpected hash conversion: %s", got) + } +} + +func TestBuildExportPlanV2FromSQLite(t *testing.T) { + inputDir := filepath.Clean("./../resources/app-mpr-v2") + mprPath, err := getMprPath(inputDir) + if err != nil { + t.Fatalf("getMprPath: %v", err) + } + version, err := getMprVersion(mprPath) + if err != nil || version != 2 { + t.Fatalf("expected mpr v2, got version %d err %v", version, err) + } + + plan, err := buildExportPlanV2(inputDir, mprPath) + if err != nil { + t.Fatalf("buildExportPlanV2() error: %v", err) + } + if len(plan.Documents) == 0 { + t.Fatal("expected documents in export plan") + } + if len(plan.unitCache) == 0 { + t.Fatal("expected structure units cached during plan build") + } + for _, doc := range plan.Documents { + if doc.ContentsHash == "" { + t.Fatalf("document %s missing ContentsHash from SQLite", doc.UnitID) + } + if _, ok := plan.mxunitPaths[doc.UnitID]; !ok { + t.Fatalf("document %s missing mxunit path", doc.UnitID) + } + if _, ok := plan.unitCache[doc.UnitID]; ok { + t.Fatalf("document %s should not be cached during plan build", doc.UnitID) + } + } +} diff --git a/serve/serve.go b/serve/serve.go index c979556..706525b 100644 --- a/serve/serve.go +++ b/serve/serve.go @@ -52,6 +52,7 @@ func runServe(cmd *cobra.Command, args []string) { } lint.SetConfig(config) configureCacheForServe(config, projectDir) + mpr.ConfigureExportConcurrency(config.Export.Concurrency) inputDirectory := config.ProjectDirectory outputDirectory := config.Modelsource @@ -373,6 +374,7 @@ func configureCacheForServe(config *lint.Config, projectDir string) { lint.SetCacheDirectory(filepath.Join(cacheBase, "lint")) mpr.SetPersistentYAMLCacheDirectory(filepath.Join(cacheBase, "mpr-v2-yaml")) mpr.SetPersistentYAMLCacheEnabled(boolValue(config.Cache.Enable, true)) + mpr.SetExportManifestPath(filepath.Join(cacheBase, "export-manifest.json")) } // addDirsRecursive adds all directories recursively to the watcher except the output directory From 1396e56a5136227dedb60c82ef4412fcc56f1cad Mon Sep 17 00:00:00 2001 From: Xiwen Cheng Date: Wed, 8 Jul 2026 00:19:26 +0200 Subject: [PATCH 5/5] fix concurrency bug --- mpr/export_stream.go | 17 +++++++++++++---- mpr/export_stream_test.go | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 4 deletions(-) diff --git a/mpr/export_stream.go b/mpr/export_stream.go index a1fe2fb..b57ddb9 100644 --- a/mpr/export_stream.go +++ b/mpr/export_stream.go @@ -46,7 +46,7 @@ type exportPlan struct { mxunitPaths map[string]string manifest *exportManifest manifestPath string - manifestMu sync.Mutex + manifestMu sync.RWMutex Close func() error } @@ -434,8 +434,17 @@ func (p *exportPlan) recordManifestEntry(unitID string, entry exportManifestEntr p.manifest.Entries[unitID] = entry } +func (p *exportPlan) lookupManifestEntry(unitID, contentsHash string) (exportManifestEntry, bool) { + p.manifestMu.RLock() + defer p.manifestMu.RUnlock() + if p.manifest == nil { + return exportManifestEntry{}, false + } + return p.manifest.entryFor(unitID, contentsHash) +} + func (p *exportPlan) tryFastSkipExport(document exportDocumentDescriptor, outputDirectory string, raw bool) (bool, error) { - entry, ok := p.manifest.entryFor(document.UnitID, document.ContentsHash) + entry, ok := p.lookupManifestEntry(document.UnitID, document.ContentsHash) if !ok || entry.RelativePath == "" { return false, nil } @@ -458,7 +467,7 @@ func (p *exportPlan) tryFastSkipExport(document exportDocumentDescriptor, output } func (p *exportPlan) tryExportFromYAMLCache(document exportDocumentDescriptor, outputDirectory string, raw bool, dirCache *exportDirCache) (string, bool, error) { - entry, ok := p.manifest.entryFor(document.UnitID, document.ContentsHash) + entry, ok := p.lookupManifestEntry(document.UnitID, document.ContentsHash) if !ok || entry.RelativePath == "" { return "", false, nil } @@ -490,7 +499,7 @@ func exportDocument(plan *exportPlan, document exportDocumentDescriptor, outputD doc := document if doc.Name == "" || doc.Type == "" { - if entry, ok := plan.manifest.entryFor(doc.UnitID, doc.ContentsHash); ok { + if entry, ok := plan.lookupManifestEntry(doc.UnitID, doc.ContentsHash); ok { doc.Name = entry.Name doc.Type = entry.Type if doc.Path == "" { diff --git a/mpr/export_stream_test.go b/mpr/export_stream_test.go index 060143a..451773b 100644 --- a/mpr/export_stream_test.go +++ b/mpr/export_stream_test.go @@ -1,8 +1,10 @@ package mpr import ( + "fmt" "os" "path/filepath" + "sync" "testing" ) @@ -24,6 +26,42 @@ func TestEffectiveExportConcurrency(t *testing.T) { } } +func TestExportPlanConcurrentManifestAccess(t *testing.T) { + t.Parallel() + + plan := &exportPlan{ + manifest: newExportManifest(), + } + + const workers = 16 + const iterations = 100 + var wg sync.WaitGroup + wg.Add(workers * 2) + + for i := 0; i < workers; i++ { + unitID := fmt.Sprintf("unit-%d", i) + go func(id string) { + defer wg.Done() + for n := 0; n < iterations; n++ { + plan.recordManifestEntry(id, exportManifestEntry{ + Name: id, + ContentsHash: "hash", + RelativePath: id + ".yaml", + }) + } + }(unitID) + + go func(id string) { + defer wg.Done() + for n := 0; n < iterations; n++ { + plan.lookupManifestEntry(id, "hash") + } + }(unitID) + } + + wg.Wait() +} + func TestExportPlanLoadUsesCache(t *testing.T) { plan := &exportPlan{ unitCache: map[string]cachedUnitContent{