diff --git a/.github/scripts/batch_index.sh b/.github/scripts/batch_index.sh new file mode 100755 index 0000000..69ac9fc --- /dev/null +++ b/.github/scripts/batch_index.sh @@ -0,0 +1,62 @@ +set -e + +# Similar to dry_run.sh, except actually builds exactly one batch group. +# This way CI can spread the full build across multiple jobs, keeping the +# total time reasonable. +batch_index=$1 + +# -f not -x since downloaded exe may not have executable permissions. +if [[ -f ./bin/clc-stackage ]]; then + echo "*** ./bin/clc-stackage exists, not re-installing ***" + + # May need to add permissions, if this exe was downloaded + chmod a+x ./bin/clc-stackage +else + echo "*** Updating cabal ***" + cabal update + + echo "*** Installing clc-stackage ***" + cabal install exe:clc-stackage --installdir=./bin --overwrite-policy=always +fi + +if [[ -d output ]]; then + rm -r output +fi + +echo "*** Building with --batch-index $batch_index ***" + +set +e + +./bin/clc-stackage \ + --batch 200 \ + --batch-index $batch_index \ + --cabal-options="--semaphore" \ + --cleanup off + +ec=$? + +if [[ $ec != 0 ]]; then + echo "*** clc-stackage failed ***" +else + echo "*** clc-stackage succeeded ***" +fi + +# Print out the logs + the packages we built, in case it is useful e.g. +# what did CI actually do. +if [[ -f generated/generated.cabal ]]; then + echo "*** Printing generated cabal file ***" + cat generated/generated.cabal +else + echo "*** No generated/generated.cabal ***" +fi + +if [[ -f generated/cabal.project.local ]]; then + echo "*** Printing generated cabal.project.local file ***" + cat generated/cabal.project.local +else + echo "*** No generated/cabal.project.local ***" +fi + +.github/scripts/print_logs.sh + +exit $ec diff --git a/.github/scripts/dry_run.sh b/.github/scripts/dry_run.sh index a2243b2..67f05c7 100755 --- a/.github/scripts/dry_run.sh +++ b/.github/scripts/dry_run.sh @@ -1,15 +1,19 @@ set -e -echo "*** Updating cabal ***" - -cabal update +if [[ -f ./bin/clc-stackage ]]; then + echo "*** ./bin/clc-stackage exists, not re-installing ***" + chmod a+x ./bin/clc-stackage +else + echo "*** Updating cabal ***" + cabal update -echo "*** Installing clc-stackage ***" + # --overwrite-policy=always and deleting output/ are unnecessary for CI since + # this script will only be run one time, but it's helpful when we are + # testing the script locally. -# --overwrite-policy=always and deleting output/ are unnecessary for CI since -# this script will only be run one time, but it's helpful when we are -# testing the script locally. -cabal install exe:clc-stackage --installdir=./bin --overwrite-policy=always + echo "*** Installing clc-stackage ***" + cabal install exe:clc-stackage --installdir=./bin --overwrite-policy=always +fi if [[ -d output ]]; then rm -r output @@ -18,7 +22,7 @@ fi echo "*** Building all with --dry-run ***" set +e -./bin/clc-stackage --batch 100 --cabal-options="--dry-run" +./bin/clc-stackage --batch 200 --cabal-options="--dry-run" ec=$? diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1de8319..ecc0b4a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -26,7 +26,7 @@ jobs: - "windows-latest" runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - uses: haskell-actions/setup@v2 with: # Should be the current stackage nightly, though this will likely go @@ -57,7 +57,7 @@ jobs: if: ${{ failure() && steps.functional.conclusion == 'failure' }} shell: bash run: .github/scripts/print_logs.sh - nix: + dry-run: strategy: fail-fast: false matrix: @@ -65,13 +65,86 @@ jobs: - "ubuntu-latest" runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Setup nix - uses: cachix/install-nix-action@v30 + uses: cachix/install-nix-action@v31 with: github_access_token: ${{ secrets.GITHUB_TOKEN }} nix_path: nixpkgs=channel:nixos-unstable - name: Dry run run: nix develop .#ci -Lv -c bash -c '.github/scripts/dry_run.sh' + + # Upload installed binary so that build-batch does not need to re-install + # it. + - name: Upload clc-stackage binary + uses: actions/upload-artifact@v7 + with: + name: clc-stackage-binary + path: ./bin/clc-stackage + retention-days: 1 + + # Uses jq's 'range(m; n)' operator to create list of indexes from [m, n) + # for the build-batch job. Slightly nicer than manually listing all of them. + build-batch-indexes: + runs-on: "ubuntu-latest" + outputs: + indexes: ${{ steps.set-batch-indexes.outputs.indexes }} + steps: + - id: set-batch-indexes + run: echo "indexes=$(jq -cn '[range(1; 19)]')" >> $GITHUB_OUTPUT + + # Ideally CI would run a job that actually builds all packages, but this + # can take a very long time, potentially longer than github's free CI limits + # (last time checked: 5.5 hrs). + # + # What we can do instead, is perform the usual batch process of dividing the + # package set into groups, then have a different job build each group. + # This does /not/ run up against github's free CI limits. + # + # To do this, we have the script batch_index.sh divide the package set into + # groups, per --batch. Then, using github's matrix strategy, have each + # job build only a specific group by passing its index as --batch-index. + # + # In other words, each job runs + # + # clc-stackage --batch N --batch-index k + # + # where k is matrix.index, hence each building a different group. + # The only other consideration we have, then, is to make sure we have enough + # indices to cover the whole package set. + # + # Currently, we choose --batch to be 200, and the total package set is + # around 3400, which is filtered to about 3100 packages to build. We thus + # need at least ceiling(3100 / 200) = 16 indexes to cover this. + # + # There is no harm in going overboard e.g. if we have an index that is out of + # range, that job will simply end with a warning message. We should + # therefore err on the side of adding too many indices, rather than too few. + build-batch: + needs: [build-batch-indexes, dry-run] + strategy: + fail-fast: false + matrix: + index: ${{ fromJSON(needs.build-batch-indexes.outputs.indexes) }} + name: Batch group ${{ matrix.index }} + runs-on: "ubuntu-latest" + steps: + - uses: actions/checkout@v6 + + - name: Setup nix + uses: cachix/install-nix-action@v31 + with: + github_access_token: ${{ secrets.GITHUB_TOKEN }} + nix_path: nixpkgs=channel:nixos-unstable + + # Download clc-stackage binary from dry-run job. + - name: Download binary + uses: actions/download-artifact@v7 + with: + name: clc-stackage-binary + path: ./bin + + - name: Build + run: nix develop .#ci -Lv -c bash -c '.github/scripts/batch_index.sh ${{ matrix.index }}' diff --git a/README.md b/README.md index a9c6def..b48361b 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,15 @@ The procedure is as follows: with-compiler: /home/ghc/_build/stage1/bin/ghc ``` -6. Run `clc-stackage` and wait for a long time. See [below](#the-clc-stackage-exe) for more details. +6. Add your custom GHC to the PATH e.g. + + ``` + export PATH=/home/ghc/_build/stage1/bin/:$PATH + ``` + + Nix users can uncomment (and modify) this line in the `flake.nix`. + +7. Run `clc-stackage` and wait for a long time. See [below](#the-clc-stackage-exe) for more details. * On a recent Macbook Air it takes around 12 hours, YMMV. * You can interrupt `cabal` at any time and rerun again later. @@ -51,14 +59,14 @@ The procedure is as follows: $ watch -n 10 "grep -Eo 'Completed|^ -' output/logs/current-build/stdout.log | sort -r | uniq -c | awk '{print \$1}'" ``` -7. If any packages fail to compile: +8. If any packages fail to compile: * copy them locally using `cabal unpack`, * patch to confirm with your proposal, * link them from `packages` section of `cabal.project`, * return to Step 6. -8. When everything finally builds, get back to CLC with a list of packages affected and patches required. +9. When everything finally builds, get back to CLC with a list of packages affected and patches required. ### Troubleshooting @@ -68,7 +76,7 @@ Because we build with `nightly` and are at the mercy of cabal's constraint solve - `p` requires a new system dependency (e.g. a C library). - `p` is an executable. - - `p` depends on a package in [./excluded_pkgs.jsonc](excluded_pkgs.jsonc). + - `p` depends on an excluded package in [./package_index.jsonc](package_index.jsonc). - A cabal flag is set in a way that breaks the build. For example, our snapshot requires that the `bson` library does *not* have its `_old-network` flag set, as this will cause a build error with our version of `network`. This flag is automatic, so we have to force it in `generated/cabal.project` with `constraints: bson -_old-network`. @@ -87,7 +95,7 @@ We attempt to mitigate such issues by: Nevertheless, it is still possible for issues to slip through. When a package `p` fails to build for some reason, we should first: -- Verify that `p` is not in `excluded_pkgs.jsonc`. If it is, nightly probably pulled in some new reverse-dependency `q` that should be added to `excluded_pkgs.jsonc`. +- Verify that `p` is not in `package_index.excluded`. If it is, nightly probably pulled in some new reverse-dependency `q` that should be added to `package_index.excluded`. - Verify that `p` does not have cabal flags that can affect dependencies / API. @@ -95,9 +103,9 @@ Nevertheless, it is still possible for issues to slip through. When a package `p In general, user mitigations for solver / build problems include: -- Adding `p` to `excluded_pkgs.jsonc`. Note that `p` will still be built if it is a (transitive) dependency of some other package in the snapshot, but will not have its exact bounds written to `cabal.project.local`. +- Adding `p` to `package_index.excluded`. Note that `p` will still be built if it is a (transitive) dependency of some other package in the snapshot, but will not have its exact bounds written to `cabal.project.local`. -- Manually downloading a snapshot (e.g. `https://www.stackage.org/nightly/cabal.config`), changing / removing the offending package(s), and supplying the file with the `--snapshot-path` param. Like `excluded_pkgs.jsonc`, take care that the problematic package is not a (transitive) dependency of something in the snapshot. +- Manually downloading a snapshot (e.g. `https://www.stackage.org/nightly/cabal.config`), changing / removing the offending package(s), and supplying the file with the `--snapshot-path` param. Like `package_index.jsonc`, take care that the problematic package is not a (transitive) dependency of something in the snapshot. - Adding constraints to `generated/cabal.project` e.g. flags or version constraints like `constraints: filepath > 1.5`. @@ -111,7 +119,7 @@ In general, user mitigations for solver / build problems include: compiler = pkgs.haskell.packages.ghc; ``` - can be a useful guide as to which GHC was last tested, as CI uses this ghc to build everything with `--dry-run`, which should report solver errors (e.g. bounds) at the very least. + can be a useful guide as to which GHC was last tested, as CI uses this ghc to build everything. - If you encounter an error that you think indicates a problem with the configuration here (e.g. new package needs to be excluded, new constraint added), please open an issue. While that is being resolved, the mitigations from the [previous section](#troubleshooting) may be useful. @@ -189,13 +197,3 @@ For Linux based systems, there's a provided `flake.nix` and `shell.nix` to get a with an approximation of the required dependencies (cabal itself, C libs) to build `clc-stackage`. Note that it is not actively maintained, so it may require some tweaking to get working, and conversely, it may have some redundant dependencies. - -## Misc - -* Your custom GHC will need to be on the PATH to build the `stack` library e.g. - - ``` - export PATH=/home/ghc/_build/stage1/bin/:$PATH - ``` - - Nix users can uncomment (and modify) this line in the `flake.nix`. diff --git a/dev.md b/dev.md index 186cfbd..3a5cdab 100644 --- a/dev.md +++ b/dev.md @@ -20,7 +20,7 @@ The `clc-stackage` library is namespaced by functionality: ### parser -`CLC.Stackage.Parser` contains the parsing functionality. In particular, `parser` is responsible for querying stackage's REST endpoint and retrieving the package set. That package set is then filtered according to [excluded_pkgs.json](excluded_pkgs.json). The primary function is: +`CLC.Stackage.Parser` contains the parsing functionality. In particular, `parser` is responsible for querying stackage's REST endpoint and retrieving the package set. That package set is then filtered according to [package_index.jsonc](package_index.jsonc). The primary function is: ```haskell -- CLC.Stackage.Parser @@ -77,13 +77,11 @@ The executable that actually runs. This is a very thin wrapper over `runner`, wh `clc-stackage` is based on `nightly` -- which changes automatically -- meaning we do not necessarily have to do anything when a new (minor) snapshot is released. On the other hand, *major* snapshot updates will almost certainly bring in new packages that need to be excluded, so there are some general "update steps" we will want to take: -1. Modify [excluded_pkgs.json](excluded_pkgs.json) as needed. That is, updating the snapshot major version will probably bring in some new packages that we do not want. The update process is essentially trial-and-error i.e. run `clc-stackage` as normal, and later add any failing packages that should be excluded. +1. Modify [package_index.jsonc](package_index.jsonc) as needed. That is, updating the snapshot major version will probably bring in some new packages that we do not want. The update process is essentially trial-and-error i.e. run `clc-stackage` as normal, and later add any failing packages to `package_index.excluded` that should be excluded. 2. Update `ghc-version` in [.github/workflows/ci.yaml](.github/workflows/ci.yaml). -3. Update functional tests as needed i.e. exact package versions in `*golden` and `test/functional/snapshot.txt`. - -4. Optional: Update nix: +3. Optional: Update nix: - Inputs (`nix flake update`). - GHC: Update the `compiler = pkgs.haskell.packages.ghc;` line. @@ -91,7 +89,7 @@ The executable that actually runs. This is a very thin wrapper over `runner`, wh This job builds everything with `--dry-run`, so its success is a useful proxy for `clc-stackage`'s health. In other words, if the nix job fails, there is almost certainly a general issue (i.e. either a package should be excluded or new system dep is required), but if it succeeds, the package set is in pretty good shape (there may still be sporadic issues e.g. a package does not properly declare its system dependencies at config time). -5. Optional: Update `clc-stackage.cabal`'s dependencies (i.e. `cabal outdated`). +4. Optional: Update `clc-stackage.cabal`'s dependencies (i.e. `cabal outdated`). ### Verifying snapshot @@ -114,3 +112,7 @@ $ NO_CLEANUP=1 cabal test functional ``` Note that this only saves files from the _last_ test, so if you want to examine test output for a particular test, you need to run only that test. + +> [!TIP] +> +> CI has a job `build-batch` which actually builds the entire package set, hence it can be used in place of manual building / testing. Note it takes about an hour to run. diff --git a/excluded_pkgs.jsonc b/excluded_pkgs.jsonc deleted file mode 100644 index 56e60fb..0000000 --- a/excluded_pkgs.jsonc +++ /dev/null @@ -1,433 +0,0 @@ -{ - "all": [ - "agda2lagda", - "al", - "alex", - "align-audio", - "Allure", - "alsa-core", - "alsa-mixer", - "alsa-pcm", - "alsa-seq", - "ALUT", - "amqp-utils", - "arbtt", - "array", // see NOTE: [Boot packages] - // NOTE: [Boot packages] - // - // Boot packages are excluded from directly building here -- and having - // their constraints written -- because - // their versions are not stable within the same major ghc version, - // and a version mismatch will cause a build failure whenever ghc - // is in the build plan (which is true for stackage). - // - // Packages taken from: - // https://gitlab.haskell.org/ghc/ghc/-/wikis/commentary/libraries/version-history - "base", - "beam-postgres", - "bench", - "bhoogle", - "binary", // see NOTE: [Boot packages] - "bindings-libzip", - "blas-carray", - "blas-comfort-array", - "blas-ffi", - "boomwhacker", - "btrfs", - "buffer-pipe", - "bugsnag", - "bugsnag-wai", - "bugsnag-yesod", - "bytestring", // see NOTE: [Boot packages] - "c2hs", - "Cabal", // see NOTE: [Boot packages] - "cabal-clean", - "cabal-flatpak", - "cabal-install", - "cabal-install-solver", - "cabal-rpm", - "cabal-sort", - "Cabal-syntax", // see NOTE: [Boot packages] - "cabal2nix", - "calendar-recycling", - "Clipboard", - "coinor-clp", - "comfort-blas", - "comfort-fftw", - "comfort-glpk", - "containers", // see NOTE: [Boot packages] - "core-telemetry", - "countdown-numbers-game", - "cql-io", - "crackNum", - "cryptonite-openssl", - "cuda", - "cutter", - "dbcleaner", - "dbus-menu", // gtk-3 - "deepseq", // see NOTE: [Boot packages] - "diagrams-svg", - "directory", // see NOTE: [Boot packages] - "discount", - "dl-fedora", - "doctest-extract", - "drifter-postgresql", - "Ebnf2ps", - "elynx", - "emd", - "equal-files", - "essence-of-live-coding-pulse", - "exceptions", // see NOTE: [Boot packages] - "experimenter", - "fbrnch", - "fedora-haskell-tools", - "fedora-repoquery", - "fft", - "fftw-ffi", - "file-io", // see NOTE: [Boot packages] - "file-modules", - "filepath", // see NOTE: [Boot packages] - "fix-whitespace", - "flac", - "flac-picture", - "follow-file", - "freckle-app", - "freenect", - "fsnotify", - "fsnotify-conduit", - "gauge", - "gd", - "ghc", // see NOTE: [Boot packages] - "ghc-bignum", // see NOTE: [Boot packages] - "ghc-boot", // see NOTE: [Boot packages] - "ghc-boot-th", // see NOTE: [Boot packages] - "ghc-compact", // see NOTE: [Boot packages] - "ghc-core", - "ghc-experimental", // see NOTE: [Boot packages] - "ghc-heap", // see NOTE: [Boot packages] - "ghc-internal", // see NOTE: [Boot packages] - "ghc-platform", // see NOTE: [Boot packages] - "ghc-prim", - "ghc-syntax-highlighter", - "ghc-toolchain", // see NOTE: [Boot packages] - "ghci", // see NOTE: [Boot packages] - "ghostscript-parallel", - "gi-atk", - "gi-cairo", - "gi-cairo-connector", - "gi-cairo-render", - "gi-dbusmenu", - "gi-dbusmenugtk3", - "gi-freetype2", - "gi-gdk", - "gi-gdk3", - "gi-gdk4", - "gi-gdkpixbuf", - "gi-gdkx11", - "gi-gdkx113", - "gi-gdkx114", - "gi-gio", - "gi-glib", - "gi-gmodule", - "gi-gobject", - "gi-graphene", - "gi-gsk", - "gi-gtk", - "gi-gtk3", - "gi-gtk4", - "gi-gtk-hs", - "gi-gtk-layer-shell", - "gi-gtksource", - "gi-gtksource5", - "gi-harfbuzz", - "gi-javascriptcore", - "gi-javascriptcore4", - "gi-javascriptcore6", - "gi-pango", - "gi-soup", - "gi-soup2", - "gi-soup3", - "gi-vte", - "gi-webkit2", - "gi-xlib", - "git-annex", - "git-mediate", - "gl", - "gloss-examples", - "glpk-headers", - "goldplate", - "google-oauth2-jwt", - "gopher-proxy", - "group-by-date", - "gtk", - "gtk-scaling-image", - "gtk-sni-tray", - "gtk-strut", - "gtk2hs-buildtools", - "gtk3", - "H", - "hackage-cli", - "haddock-api", // see NOTE: [Boot packages] - "haddock-library", // see NOTE: [Boot packages] - "hamtsolo", - "happstack-server-tls", - "happy", - "haskeline", // see NOTE: [Boot packages] - "haskell-gi", - "haskell-gi-base", - "haskell-gi-overloading", - "haskoin-core", - "haskoin-node", - "haskoin-store", - "haskoin-store-data", - "hasql", - "hasql-dynamic-statements", - "hasql-implicits", - "hasql-interpolate", - "hasql-listen-notify", - "hasql-migration", - "hasql-notifications", - "hasql-optparse-applicative", - "hasql-pool", - "hasql-queue", - "hasql-th", - "hasql-transaction", - "haxr", - "headroom", // segfault - "hinotify", - "hkgr", - "hledger-interest", - "hledger-ui", - "hlibgit2", - "hmatrix-gsl", - "hmatrix-gsl-stats", - "hmatrix-special", - "hmm-lapack", - "hmpfr", - "hpc", // see NOTE: [Boot packages] - "hopenssl", - "hp2pretty", - "hpqtypes", - "hpqtypes-extras", - "hruby", - "hs-captcha", - "hs-GeoIP", - "hsc2hs", - "hsdns", - "hsignal", - "hsndfile", - "hsndfile-vector", - "HsOpenSSL", - "HsOpenSSL-x509-system", - "hsshellscript", - "hstatistics", - "htaglib", - "http-client-openssl", - "http-io-streams", - "http-streams", - "hw-json-simd", - "hw-kafka-client", - "hwk", - "ihaskell", - "ihaskell-hvega", - "ihs", - "Imlib", - "inline-r", - "integer-gmp", // see NOTE: [Boot packages] - "integer-simple", - "ip6addr", - "ipython-kernel", - "jack", - "jailbreak-cabal", - "java-adt", - "JuicyPixels-scale-dct", - "koji", - "koji-tool", - "krank", - "LambdaHack", - "lame", - "lapack", - "lapack-carray", - "lapack-comfort-array", - "lapack-ffi", - "lapack-ffi-tools", - "lapack-hmatrix", - "lens-regex-pcre", - "lentil", - "leveldb-haskell", - "liboath-hs", - "linear-circuit", - "linux-file-extents", - "linux-namespaces", - "lmdb", - "lsfrom", - "lzma-clib", - "magic", - "magico", - "mbox-utility", - "mega-sdist", - "microformats2-parser", // segfault - "midi-alsa", - "midi-music-box", - "misfortune", - "mmark-cli", - "mmark-ext", // ghc-syntax-highlighter - "moffy-samples-gtk3", - "moffy-samples-gtk3-run", - "moffy-samples-gtk4", - "moffy-samples-gtk4-run", - "mpi-hs", - "mpi-hs-binary", - "mpi-hs-cereal", - "mstate", - "mtl", // see NOTE: [Boot packages] - "mysql", - "mysql-json-table", - "mysql-simple", - "nanovg", - "netcode-io", - "nfc", - "NineP", - "nix-paths", - "nvvm", - "odbc", - "ods2csv", - "ogma-cli", - "opaleye", - "openssl-streams", - "OrderedBits", - "os-string", // see NOTE: [Boot packages] - "pagure-cli", - "pandoc-cli", - "parsec", // see NOTE: [Boot packages] - "parser-combinators-tests", - "pcre-heavy", - "pcre-light", - "peregrin", - "perf", - "persistent-mysql", - "persistent-postgresql", - "pg-transact", - "pkgtreediff", - "place-cursor-at", - "pontarius-xmpp", - "pontarius-xmpp-extras", - "postgresql-libpq-notify", - "postgresql-migration", - "postgresql-schema", - "postgresql-simple", - "postgresql-simple-url", - "pretty", // see NOTE: [Boot packages] - "primecount", - "process", // see NOTE: [Boot packages] - "profiterole", - "proto-lens-protobuf-types", - "psql-helpers", - "pthread", - "pulse-simple", - "rawfilepath", - "rdtsc", - "re2", - "reactive-balsa", - "reactive-jack", - "reanimate-svg", - "regex-pcre", - "rel8", - "resistor-cube", - "rex", - "rhbzquery", - "rocksdb-haskell", - "rocksdb-haskell-jprupp", - "rocksdb-query", - "rts", // see NOTE: [Boot packages] - "scrypt", - "sdl2", - "sdl2-gfx", - "sdl2-image", - "sdl2-mixer", - "sdl2-ttf", - "secp256k1-haskell", - "semaphore-compat", // see NOTE: [Boot packages] - "seqalign", - "servant-http-streams", - "servius", - "ses-html", - "shelltestrunner", - "snappy", - "sound-collage", - "soxlib", - "sphinx", - "split-record", - "sqlcli", - "sqlcli-odbc", - "sqlite-simple", - "stakhanov", // hasql - "stack-all", - "stack-clean-old", - "stack-templatizer", - "stm", // see NOTE: [Boot packages] - "stringprep", - "SVGFonts", - "swizzle", - "swizzle-lens", - "swizzle-modify", - "swizzle-set", - "sydtest-persistent-postgresql", - "synthesizer-alsa", - "taffybar", - "tasty-papi", - "template-haskell", // see NOTE: [Boot packages] - "termonad", - "terminfo", // see NOTE: [Boot packages] - "test-certs", - "text", // see NOTE: [Boot packages] - "text-icu", - "text-regex-replace", - "time", // see NOTE: [Boot packages] - "tls-debug", - "tmp-postgres", - "tmp-proc-postgres", - "transformers", // see NOTE: [Boot packages] - "ua-parser", - "uniq-deep", - "unix", // see NOTE: [Boot packages] - "users-postgresql-simple", - "validate-input", - "vector-fftw", - "visualize-type-inference", - "wai-session-postgresql", - "web3-tools", - "wild-bind-x11", - "Win32", // see NOTE: [Boot packages] - "Win32-notify", - "windns", - "X11", - "X11-xft", - "x11-xim", - "xhtml", // see NOTE: [Boot packages] - "xmonad", - "xmonad-contrib", - "xmonad-extras", - "yesod-bin", - "yi", - "yi-frontend-pango", - "yoga", - "youtube", - "zeromq4-haskell", - "zeromq4-patterns", - "zot", - "ztail" - ], - "linux": [ - "hfsevents", - "vty-windows" - ], - "osx": [ - "vty-windows" - ], - "windows": [ - "hfsevents", - "postgresql-libpq", - "vty-unix" - ] -} diff --git a/package_index.jsonc b/package_index.jsonc new file mode 100644 index 0000000..42cb5ac --- /dev/null +++ b/package_index.jsonc @@ -0,0 +1,447 @@ +{ + // All top-level keys are disjoint in the sense that a given package should + // belong to at most one. The notable exception is for packages that are + // in multiple excluded subkeys e.g. vty-windows in excluded.linux and + // excluded.osx. + + // "Excluded" packages are those that are excluded from the package set i.e. + // they will not be added to the generated/generated.cabal build-depends or + // have their versions pinned in generated/cabal.project.local. + "excluded": { + "all": [ + "agda2lagda", + "al", + "align-audio", + "Allure", + "alsa-core", + "alsa-mixer", + "alsa-pcm", + "alsa-seq", + "ALUT", + "amqp-utils", + "arbtt", + "beam-postgres", + "bench", + "bhoogle", + "bindings-libzip", + "blas-carray", + "blas-comfort-array", + "blas-ffi", + "boomwhacker", + "btrfs", + "buffer-pipe", + "bugsnag", + "bugsnag-wai", + "bugsnag-yesod", + "c2hs", + "cabal-clean", + "cabal-flatpak", + "cabal-install", + "cabal-install-solver", + "cabal-rpm", + "cabal-sort", + "cabal2nix", + "calendar-recycling", + "Clipboard", + "coinor-clp", + "comfort-blas", + "comfort-fftw", + "comfort-glpk", + "core-telemetry", + "countdown-numbers-game", + "cql-io", + "crackNum", + "cryptonite-openssl", + "cuda", + "cutter", + "dbcleaner", + "dbus-menu", // gtk-3 + "diagrams-svg", + "discount", + "dl-fedora", + "doctest-extract", + "drifter-postgresql", + "Ebnf2ps", + "elynx", + "emd", + "equal-files", + "essence-of-live-coding-pulse", + "experimenter", + "fbrnch", + "fedora-haskell-tools", + "fedora-repoquery", + "fft", + "fftw-ffi", + "file-modules", + "fix-whitespace", + "flac", + "flac-picture", + "follow-file", + "freckle-app", + "freenect", + "fsnotify", + "fsnotify-conduit", + "gauge", + "gd", + "ghc", + "ghc-bignum", + "ghc-boot", + "ghc-boot-th", + "ghc-compact", + "ghc-core", + "ghc-experimental", + "ghc-heap", + "ghc-internal", + "ghc-platform", + "ghc-prim", + "ghc-syntax-highlighter", + "ghc-toolchain", + "ghci", + "ghostscript-parallel", + "gi-atk", + "gi-cairo", + "gi-cairo-connector", + "gi-cairo-render", + "gi-dbusmenu", + "gi-dbusmenugtk3", + "gi-freetype2", + "gi-gdk", + "gi-gdk3", + "gi-gdk4", + "gi-gdkpixbuf", + "gi-gdkx11", + "gi-gdkx113", + "gi-gdkx114", + "gi-gio", + "gi-glib", + "gi-gmodule", + "gi-gobject", + "gi-graphene", + "gi-gsk", + "gi-gtk", + "gi-gtk3", + "gi-gtk4", + "gi-gtk-hs", + "gi-gtk-layer-shell", + "gi-gtksource", + "gi-gtksource5", + "gi-harfbuzz", + "gi-javascriptcore", + "gi-javascriptcore4", + "gi-javascriptcore6", + "gi-pango", + "gi-soup", + "gi-soup2", + "gi-soup3", + "gi-vte", + "gi-webkit2", + "gi-xlib", + "git-annex", + "git-mediate", + "gl", + "gloss-examples", + "glpk-headers", + "goldplate", + "google-oauth2-jwt", + "gopher-proxy", + "group-by-date", + "gtk", + "gtk-scaling-image", + "gtk-sni-tray", + "gtk-strut", + "gtk2hs-buildtools", + "gtk3", + "H", + "hackage-cli", + "hamtsolo", + "happstack-server-tls", + "haskell-gi", + "haskell-gi-base", + "haskell-gi-overloading", + "haskoin-core", + "haskoin-node", + "haskoin-store", + "haskoin-store-data", + "hasql", + "hasql-dynamic-statements", + "hasql-implicits", + "hasql-interpolate", + "hasql-listen-notify", + "hasql-migration", + "hasql-notifications", + "hasql-optparse-applicative", + "hasql-pool", + "hasql-queue", + "hasql-th", + "hasql-transaction", + "haxr", + "headroom", // segfault + "hinotify", + "hkgr", + "hledger-interest", + "hledger-ui", + "hlibgit2", + "hmatrix-gsl", + "hmatrix-gsl-stats", + "hmatrix-special", + "hmm-lapack", + "hmpfr", + "hopenssl", + "hp2pretty", + "hpqtypes", + "hpqtypes-extras", + "hruby", + "hs-captcha", + "hs-GeoIP", + "hsc2hs", + "hsdns", + "hsignal", + "hsndfile", + "hsndfile-vector", + "HsOpenSSL", + "HsOpenSSL-x509-system", + "hsshellscript", + "hstatistics", + "htaglib", + "http-client-openssl", + "http-io-streams", + "http-streams", + "hw-json-simd", + "hw-kafka-client", + "hwk", + "ihaskell", + "ihaskell-hvega", + "ihs", + "Imlib", + "inline-r", + "integer-simple", + "ip6addr", + "ipython-kernel", + "jack", + "jailbreak-cabal", + "java-adt", + "JuicyPixels-scale-dct", + "koji", + "koji-tool", + "krank", + "LambdaHack", + "lame", + "lapack", + "lapack-carray", + "lapack-comfort-array", + "lapack-ffi", + "lapack-ffi-tools", + "lapack-hmatrix", + "lens-regex-pcre", + "lentil", + "leveldb-haskell", + "liboath-hs", + "linear-circuit", + "linux-file-extents", + "linux-namespaces", + "lmdb", + "lsfrom", + "lzma-clib", + "magic", + "magico", + "mbox-utility", + "mega-sdist", + "microformats2-parser", // segfault + "midi-alsa", + "midi-music-box", + "misfortune", + "mmark-cli", + "mmark-ext", // ghc-syntax-highlighter + "moffy-samples-gtk3", + "moffy-samples-gtk3-run", + "moffy-samples-gtk4", + "moffy-samples-gtk4-run", + "mpi-hs", + "mpi-hs-binary", + "mpi-hs-cereal", + "mstate", + "mysql", + "mysql-json-table", + "mysql-simple", + "nanovg", + "netcode-io", + "nfc", + "NineP", + "nix-paths", + "nvvm", + "odbc", + "ods2csv", + "ogma-cli", + "opaleye", + "openssl-streams", + "OrderedBits", + "pagure-cli", + "pandoc-cli", + "parser-combinators-tests", + "pcre-heavy", + "pcre-light", + "peregrin", + "perf", + "persistent-mysql", + "persistent-postgresql", + "pg-transact", + "pkgtreediff", + "place-cursor-at", + "pontarius-xmpp", + "pontarius-xmpp-extras", + "postgresql-libpq-notify", + "postgresql-migration", + "postgresql-schema", + "postgresql-simple", + "postgresql-simple-url", + "primecount", + "profiterole", + "proto-lens-protobuf-types", + "psql-helpers", + "pthread", + "pulse-simple", + "rawfilepath", + "rdtsc", + "re2", + "reactive-balsa", + "reactive-jack", + "reanimate-svg", + "regex-pcre", + "rel8", + "resistor-cube", + "rex", + "rhbzquery", + "rocksdb-haskell", + "rocksdb-haskell-jprupp", + "rocksdb-query", + "rts", + "scrypt", + "sdl2", + "sdl2-gfx", + "sdl2-image", + "sdl2-mixer", + "sdl2-ttf", + "secp256k1-haskell", + "seqalign", + "servant-http-streams", + "servius", + "ses-html", + "shelltestrunner", + "snappy", + "sound-collage", + "soxlib", + "sphinx", + "split-record", + "sqlcli", + "sqlcli-odbc", + "sqlite-simple", + "stakhanov", // hasql + "stack-all", + "stack-clean-old", + "stack-templatizer", + "stringprep", + "SVGFonts", + "swizzle", + "swizzle-lens", + "swizzle-modify", + "swizzle-set", + "sydtest-persistent-postgresql", + "synthesizer-alsa", + "taffybar", + "tasty-papi", + "termonad", + "test-certs", + "text-icu", + "text-regex-replace", + "tls-debug", + "tmp-postgres", + "tmp-proc-postgres", + "ua-parser", + "uniq-deep", + "unix", + "users-postgresql-simple", + "validate-input", + "vector-fftw", + "visualize-type-inference", + "wai-session-postgresql", + "web3-tools", + "wild-bind-x11", + "Win32", + "Win32-notify", + "windns", + "X11", + "X11-xft", + "x11-xim", + "xmonad", + "xmonad-contrib", + "xmonad-extras", + "yesod-bin", + "yi", + "yi-frontend-pango", + "yoga", + "youtube", + "zeromq4-haskell", + "zeromq4-patterns", + "zot", + "ztail" + ], + "linux": [ + "hfsevents", + "vty-windows" + ], + "osx": [ + "vty-windows" + ], + "windows": [ + "hfsevents", + "postgresql-libpq", + "vty-unix" + ] + }, + // "Excluded pinned" are packages like excluded -- i.e. also excluded from + // build-depends -- but we nevertheless want pinned, for reproducibility. + // This makes sense for transitive dependencies that we cannot build + // directly e.g. exes used in setup. + "excluded_pinned": [ + "alex", + "happy" + ], + // "Unpinned" are packages that we /do/ want to build (i.e. not part of + // the "excluded" group) but do not want to pin exactly e.g. boot packages, + // because exact pins would prevent building with multiple GHCs. + // + // Boot packages taken from: + // https://gitlab.haskell.org/ghc/ghc/-/wikis/commentary/libraries/version-history + "unpinned": [ + "array", + "base", + "binary", + "bytestring", + "Cabal", + "Cabal-syntax", + "containers", + "deepseq", + "directory", + "exceptions", + "file-io", + "filepath", + "haddock-api", + "haddock-library", + "haskeline", + "hpc", + "integer-gmp", + "mtl", + "os-string", + "parsec", + "pretty", + "process", + "semaphore-compat", + "stm", + "template-haskell", + "terminfo", + "text", + "time", + "transformers", + "xhtml" + ] +} diff --git a/src/CLC/Stackage/Builder/Env.hs b/src/CLC/Stackage/Builder/Env.hs index a6c565e..5a27e6f 100644 --- a/src/CLC/Stackage/Builder/Env.hs +++ b/src/CLC/Stackage/Builder/Env.hs @@ -34,6 +34,10 @@ data BuildEnv = MkBuildEnv { -- | If we have @Just n@, 'packagesToBuild' will be split into groups of at most -- size @n@. If @Nothing@, the entire set will be built in one go. batch :: Maybe Int, + -- | 1-based index for building the Nth package group only, according to + -- --batch. Intended for CI use, where building all groups takes too much + -- time. + batchIndex :: Maybe Int, -- | Build arguments for cabal. buildArgs :: [String], -- | Optional path to cabal executable. diff --git a/src/CLC/Stackage/Builder/Writer.hs b/src/CLC/Stackage/Builder/Writer.hs index 48787ea..1b87f4a 100644 --- a/src/CLC/Stackage/Builder/Writer.hs +++ b/src/CLC/Stackage/Builder/Writer.hs @@ -5,11 +5,12 @@ module CLC.Stackage.Builder.Writer where import CLC.Stackage.Builder.Batch (PackageGroup (unPackageGroup)) +import CLC.Stackage.Parser (PackageSet (extraPins, packageList, unpinned)) import CLC.Stackage.Utils.IO qualified as IO -import CLC.Stackage.Utils.Package (Package) import CLC.Stackage.Utils.Package qualified as Package import CLC.Stackage.Utils.Paths qualified as Paths import Data.List.NonEmpty qualified as NE +import Data.Set qualified as Set import Data.Text (Text) import Data.Text qualified as T import Data.Text.Encoding qualified as TEnc @@ -27,13 +28,32 @@ import Data.Text.Encoding qualified as TEnc -- By writing the entire (exact) dependency set into the cabal.project.local's -- constraints section, we ensure the same version of aeson is used every time -- it is a (transitive) dependency. -writeCabalProjectLocal :: [Package] -> IO () -writeCabalProjectLocal pkgs = IO.writeBinaryFile path constraintsSrc +writeCabalProjectLocal :: PackageSet -> IO () +writeCabalProjectLocal packageSet = IO.writeBinaryFile path constraintsSrc where path = Paths.generatedCabalProjectLocalPath constraintsSrc = TEnc.encodeUtf8 constraintsTxt constraintsTxt = T.unlines $ "constraints:" : constraints - constraints = (\p -> " " <> Package.toCabalConstraintsText p) <$> pkgs + -- Use any prefix so that this applies to setup too e.g. happy + constraints = (\p -> " any." <> Package.toCabalConstraintsText p) <$> constrainedPkgs + + -- In addition to all of the normal packages whose constraints we want to + -- write, we also need to: + -- + -- - Add in extraPins. + -- - Remove any unpinned. + constrainedPkgs = + Set.toList + . removeUnpinned + . addExtraPins + . Set.fromList + $ packageSet.packageList + + addExtraPins = Set.union (Set.fromList packageSet.extraPins) + + removeUnpinned = + Set.filter + (\p -> Set.notMember p.name (Set.fromList packageSet.unpinned)) -- | Writes the package set to a cabal file for building. This will be called -- for each group we want to build. diff --git a/src/CLC/Stackage/Parser.hs b/src/CLC/Stackage/Parser.hs index bc0669e..b652589 100644 --- a/src/CLC/Stackage/Parser.hs +++ b/src/CLC/Stackage/Parser.hs @@ -3,7 +3,9 @@ module CLC.Stackage.Parser ( -- * Retrieving packages + PackageSet (..), getPackageList, + packageListToSet, -- * Misc helpers printPackageList, @@ -25,11 +27,11 @@ import CLC.Stackage.Utils.OS (Os (Linux, Osx, Windows)) import CLC.Stackage.Utils.OS qualified as OS import CLC.Stackage.Utils.Package (Package) import CLC.Stackage.Utils.Package qualified as Package -import Control.Monad (when) +import Control.Exception (throwIO) +import Control.Monad (unless, when) import Data.Aeson (FromJSON, ToJSON) import Data.Foldable (for_) import Data.Maybe (fromMaybe) -import Data.Set (Set) import Data.Set qualified as Set import Data.Text (Text) import Data.Text qualified as T @@ -41,12 +43,18 @@ import System.OsPath (OsPath, osp) import System.Process qualified as P import Text.ParserCombinators.ReadP qualified as ReadP --- | Retrieves the list of packages, based on +-- | Retrieves the 'PackageSet', based on -- 'CLC.Stackage.Parser.API.stackageUrl'. -getPackageList :: Logging.Handle -> Maybe OsPath -> IO [Package] +getPackageList :: Logging.Handle -> Maybe OsPath -> IO PackageSet getPackageList hLogger msnapshotPath = do response <- getStackageResponse hLogger msnapshotPath - getPackageListByOs hLogger response OS.currentOs + packageListToSet hLogger response.packages + +-- | Given a list of packages, returns a 'PackageSet' i.e. all constraints +-- and filtered packages, according to package_index.jsonc +packageListToSet :: Logging.Handle -> [Package] -> IO PackageSet +packageListToSet hLogger packages = + getPackageListByOs hLogger packages OS.currentOs -- | Prints the package list to a file. printPackageList :: Logging.Handle -> Maybe OsPath -> Maybe Os -> IO () @@ -68,15 +76,41 @@ printPackageList hLogger msnapshotPath mOs = do -- | Retrieves the package list formatted to text. getPackageListByOsFmt :: Logging.Handle -> StackageResponse -> Os -> IO [Text] getPackageListByOsFmt hLogger response os = do - ps <- getPackageListByOs hLogger response os - pure $ Package.toDisplayName <$> ps + ps <- getPackageListByOs hLogger response.packages os + pure $ Package.toDisplayName <$> ps.packageList + +-- | PackageSet corresponds to the package set after filtering i.e. we +-- get some list of packages either from Stackage or --snapshot-path, +-- then we apply filtering according to package_index.jsonc +data PackageSet = MkPackageSet + { -- | Extra packages that are not in packageList, but nevertheless we want + -- to pin e.g. because they are transitive dependencies. + extraPins :: [Package], + -- | Package set after filtering out excluded packages. + packageList :: [Package], + -- | Package in packageList that we do not want to pin. + unpinned :: [Text] + } + deriving stock (Eq, Show) -- | Helper in case we want to see what the package set for a given OS is. -getPackageListByOs :: Logging.Handle -> StackageResponse -> Os -> IO [Package] -getPackageListByOs hLogger response os = do - excludedPkgs <- getExcludedPkgs os - let filterExcluded = flip Set.notMember excludedPkgs . (.name) - packages = filter filterExcluded response.packages +getPackageListByOs :: Logging.Handle -> [Package] -> Os -> IO PackageSet +getPackageListByOs hLogger packageList os = do + packageIndex <- getPackageIndex hLogger + + -- Setup extra pins i.e. all p in packageList in packageIndex.excluded_pinned. + let excludedPinnedCfg = Set.fromList packageIndex.excluded_pinned + isExcludedPinned = flip Set.member excludedPinnedCfg . (.name) + excludedPinned = filter isExcludedPinned packageList + + -- Setup excluded. This is packageIndex.excluded + + -- packageIndex.excluded_pinned. + let excluded = + Set.fromList (packageIndex.excluded.all ++ osSel packageIndex.excluded) + `Set.union` excludedPinnedCfg + + isNotExcluded = flip Set.notMember excluded . (.name) + packages = filter isNotExcluded packageList msg = mconcat [ "Filtered to ", @@ -87,7 +121,18 @@ getPackageListByOs hLogger response os = do ] Logging.putTimeInfoStr hLogger msg - pure packages + pure $ + MkPackageSet + { extraPins = excludedPinned, + packageList = packages, + unpinned = packageIndex.unpinned + } + where + osSel :: Excluded -> [Text] + osSel = case os of + Linux -> (.linux) + Osx -> (.osx) + Windows -> (.windows) getStackageResponse :: Logging.Handle -> Maybe OsPath -> IO StackageResponse getStackageResponse hLogger msnapshotPath = do @@ -123,28 +168,108 @@ getStackageResponse hLogger msnapshotPath = do pure response -getExcludedPkgs :: Os -> IO (Set Text) -getExcludedPkgs os = do +getPackageIndex :: Logging.Handle -> IO PackageIndex +getPackageIndex hLogger = do contents <- Ex.throwLeft . Utils.stripComments =<< IO.readBinaryFile path - excluded <- case JSON.decode contents of + packageIndex <- case JSON.decode @PackageIndex contents of Left err -> fail err Right x -> pure x - pure $ Set.fromList (excluded.all ++ osSel excluded) + verifyIndex packageIndex + + pure packageIndex where - path = [osp|excluded_pkgs.jsonc|] + path = [osp|package_index.jsonc|] - osSel :: Excluded -> [Text] - osSel = case os of - Linux -> (.linux) - Osx -> (.osx) - Windows -> (.windows) + -- Verifies index properties. + verifyIndex index = do + -- Each key should not have duplicates. A violation is mostly harmless, + -- presumably an oversight. + warnDuplicates "excluded.all" index.excluded.all + warnDuplicates "excluded.linux" index.excluded.linux + warnDuplicates "excluded.osx" index.excluded.osx + warnDuplicates "excluded.windows" index.excluded.windows + + warnDuplicates "excluded_pinned" index.excluded_pinned + warnDuplicates "unpinned" index.unpinned + + let allOs = Set.fromList index.excluded.all + linux = Set.fromList index.excluded.linux + osx = Set.fromList index.excluded.osx + windows = Set.fromList index.excluded.windows + + excluded = Set.unions [allOs, linux, osx, windows] + excludedPinned = Set.fromList index.excluded_pinned + unpinned = Set.fromList index.unpinned + + -- Each of these should be disjoint. Note that various excluded subkeys + -- may not be disjoint e.g. excluded.linux and excluded.osx both + -- exclude vty-windows. Hence we only check the combined excluded + -- against the other two. + -- + -- This is more serious than duplicate violations as it could lead + -- to confusing behavior e.g. if a package is in excluded_pinned + -- and unpinned. + errNonDisjoint "excluded" excluded "excluded_pinned" excludedPinned + errNonDisjoint "excluded" excluded "unpinned" unpinned + errNonDisjoint "excluded_pinned" excludedPinned "unpinned" unpinned + + errNonDisjoint xName x yName y = do + let z = Set.intersection x y + unless (Set.null z) $ do + let libs = T.intercalate ", " $ Set.toList z + msg = + mconcat + [ "package_index.jsonc: ", + xName, + " and ", + yName, + " are not disjoint: ", + libs + ] + Logging.putTimeErrStr hLogger msg + throwIO $ ExitFailure 1 + + warnDuplicates xName x = do + let (_, duplicates) = foldl' go (Set.empty, Set.empty) x + go (foundSoFar, dupes) l = + if Set.member l foundSoFar + then (foundSoFar, Set.insert l dupes) + else (Set.insert l foundSoFar, dupes) + + unless (Set.null duplicates) $ do + let libs = T.intercalate ", " $ Set.toList duplicates + msg = + mconcat + [ "package_index.jsonc: ", + xName, + " has duplicates: ", + libs + ] + Logging.putTimeWarnStr hLogger msg + +-- | Corresponds to package_index.jsonc +data PackageIndex = MkPackageIndex + { -- | Excluded packages. + excluded :: Excluded, + -- | Excluded packages that we nevertheless pin. + excluded_pinned :: [Text], + -- | Normal packages that should not be pinned. + unpinned :: [Text] + } + deriving stock (Eq, Generic, Show) + deriving anyclass (FromJSON, ToJSON) +-- | package_index.excluded. data Excluded = MkExcluded - { all :: [Text], + { -- | Packages excluded from all Os's. + all :: [Text], + -- | Packages excluded from linux. linux :: [Text], + -- | Packages excluded from osx. osx :: [Text], + -- | Packages excluded from windows. windows :: [Text] } deriving stock (Eq, Generic, Show) diff --git a/src/CLC/Stackage/Parser/Utils.hs b/src/CLC/Stackage/Parser/Utils.hs index c19ef04..c0ab84e 100644 --- a/src/CLC/Stackage/Parser/Utils.hs +++ b/src/CLC/Stackage/Parser/Utils.hs @@ -5,6 +5,7 @@ module CLC.Stackage.Parser.Utils -- * Misc isNum, + commaW8, spaceW8, ) where @@ -102,8 +103,11 @@ fslashW8 = i2w8 $ Ch.ord '/' spaceW8 :: Word8 spaceW8 = i2w8 $ Ch.ord ' ' +commaW8 :: Word8 +commaW8 = i2w8 $ Ch.ord ',' + isNum :: Word8 -> Bool -isNum w = w >= (i2w8 $ Ch.ord '0') && w <= (i2w8 $ Ch.ord '9') +isNum w = w >= i2w8 (Ch.ord '0') && w <= i2w8 (Ch.ord '9') i2w8 :: Int -> Word8 i2w8 = fromIntegral diff --git a/src/CLC/Stackage/Runner.hs b/src/CLC/Stackage/Runner.hs index 1ade3d5..a4aa246 100644 --- a/src/CLC/Stackage/Runner.hs +++ b/src/CLC/Stackage/Runner.hs @@ -8,7 +8,7 @@ where import CLC.Stackage.Builder qualified as Builder import CLC.Stackage.Builder.Env - ( BuildEnv (hLogger, progress), + ( BuildEnv (batchIndex, hLogger, progress), Progress (failuresRef), ) import CLC.Stackage.Builder.Writer qualified as Writer @@ -17,9 +17,13 @@ import CLC.Stackage.Runner.Env qualified as Env import CLC.Stackage.Utils.Logging qualified as Logging import CLC.Stackage.Utils.Package (Package) import Control.Exception (bracket, throwIO) -import Control.Monad (unless, when) +import Control.Monad (when) import Data.Foldable (for_) import Data.IORef (readIORef) +import Data.List.NonEmpty (NonEmpty) +import Data.List.NonEmpty qualified as NE +import Data.Text (Text) +import Data.Text qualified as T import System.Exit (ExitCode (ExitFailure)) import System.IO qualified as IO @@ -48,11 +52,28 @@ runModifyPackages hLogger modifyPackages = withHiddenInput $ do -- write the entire package set to the cabal.project.local's constraints Writer.writeCabalProjectLocal env.completePackageSet - unless env.noCabalUpdate $ Builder.cabalUpdate buildEnv + when env.cabalUpdate $ Builder.cabalUpdate buildEnv Logging.putTimeInfoStr buildEnv.hLogger "Starting build(s)" - for_ pkgGroupsIdx $ \(pkgGroup, idx) -> Builder.buildProject buildEnv idx pkgGroup + case buildEnv.batchIndex of + -- No batch index: normal, build all groups sequentially. + Nothing -> for_ pkgGroupsIdx $ \(pkgGroup, idx) -> + Builder.buildProject buildEnv idx pkgGroup + Just batchIndex -> + -- Some batch index: if it is in range, build that group. + case index batchIndex pkgGroupsIdx of + Just (pkgGroup, idx) -> Builder.buildProject buildEnv idx pkgGroup + Nothing -> do + let msg = + mconcat + [ "Nothing to build. Index '", + showt batchIndex, + "' is out of range for ", + showt $ NE.length pkgGroupsIdx, + " group(s)." + ] + Logging.putTimeWarnStr buildEnv.hLogger msg numErrors <- length <$> readIORef buildEnv.progress.failuresRef when (numErrors > 0) $ throwIO $ ExitFailure 1 @@ -75,3 +96,16 @@ withHiddenInput m = bracket hideInput unhideInput (const m) unhideInput (buffMode, echoMode) = do IO.hSetBuffering IO.stdin buffMode IO.hSetEcho IO.stdin echoMode + +index :: Int -> NonEmpty a -> Maybe a +index idx = go idx' . NE.toList + where + go _ [] = Nothing + go 0 (x : _) = Just x + go !n (_ : xs) = go (n - 1) xs + + -- Subtract one since the index is one-based, not zero. + idx' = idx - 1 + +showt :: (Show a) => a -> Text +showt = T.pack . show diff --git a/src/CLC/Stackage/Runner/Args.hs b/src/CLC/Stackage/Runner/Args.hs index 431a25d..797fae1 100644 --- a/src/CLC/Stackage/Runner/Args.hs +++ b/src/CLC/Stackage/Runner/Args.hs @@ -58,25 +58,29 @@ data Args = MkArgs { -- | If given, batches packages together so we build more than one. -- Defaults to batching everything together in the same group. batch :: Maybe Int, + -- | 1-based index for building the Nth package group only, according to + -- --batch. Intended for CI use, where building all groups takes too much + -- time. + batchIndex :: Maybe Int, -- | Global options to pass to cabal e.g. --store-dir. cabalGlobalOpts :: [String], -- | Options to pass to cabal e.g. --semaphore. cabalOpts :: [String], -- | Optional path to cabal executable. cabalPath :: Maybe OsPath, + -- | If true, the 'cabal update' step is run. + cabalUpdate :: Bool, + -- | Enables the cache, which saves the outcome of a run in a json file. + -- The cache is used for resuming a run that was interrupted. + cache :: Bool, + -- | If true, leaves the last generated cabal files. + cleanup :: Bool, -- | Determines if we color the logs. If 'Nothing', attempts to detect -- if colors are supported. colorLogs :: ColorLogs, -- | If true, the first group that fails to completely build stops -- clc-stackage. groupFailFast :: Bool, - -- | If true, the 'cabal update' step is skipped. - noCabalUpdate :: Bool, - -- | Disables the cache, which otherwise saves the outcome of a run in a - -- json file. The cache is used for resuming a run that was interrupted. - noCache :: Bool, - -- | If true, leaves the last generated cabal files. - noCleanup :: Bool, -- | If true, the first package that fails _within_ a package group will -- cause the entire group to fail. packageFailFast :: Bool, @@ -188,23 +192,24 @@ getArgs = OA.execParser parserInfoArgs parseCliArgs :: Parser Args parseCliArgs = ( do - ~(cabalGlobalOpts, cabalOpts, cabalPath, noCabalUpdate) <- parseCabalGroup - ~(noCache, retryFailures) <- parseCacheGroup + ~(cabalGlobalOpts, cabalOpts, cabalPath, cabalUpdate) <- parseCabalGroup + ~(cache, retryFailures) <- parseCacheGroup ~(groupFailFast, packageFailFast) <- parseFailuresGroup - ~(batch, printPackageSet, snapshotPath) <- parseMiscGroup - ~(colorLogs, noCleanup, writeLogs) <- parseOutputGroup + ~(batch, batchIndex, printPackageSet, snapshotPath) <- parseMiscGroup + ~(cleanup, colorLogs, writeLogs) <- parseOutputGroup pure $ MkArgs { batch, + batchIndex, cabalGlobalOpts, cabalOpts, cabalPath, + cabalUpdate, + cache, + cleanup, colorLogs, groupFailFast, - noCabalUpdate, - noCache, - noCleanup, packageFailFast, printPackageSet, retryFailures, @@ -220,12 +225,12 @@ parseCliArgs = <$> parseCabalGlobalOpts <*> parseCabalOpts <*> parseCabalPath - <*> parseNoCabalUpdate + <*> parseCabalUpdate parseCacheGroup = OA.parserOptionGroup "Cache options:" $ (,) - <$> parseNoCache + <$> parseCache <*> parseRetryFailures parseFailuresGroup = @@ -236,16 +241,17 @@ parseCliArgs = parseMiscGroup = OA.parserOptionGroup "Misc options:" $ - (,,) + (,,,) <$> parseBatch + <*> parseBatchIndex <*> parsePrintPackageSet <*> parseSnapshotPath parseOutputGroup = OA.parserOptionGroup "Output options:" $ (,,) - <$> parseColorLogs - <*> parseNoCleanup + <$> parseCleanup + <*> parseColorLogs <*> parseWriteLogs parseBatch :: Parser (Maybe Int) @@ -267,6 +273,22 @@ parseBatch = ] ) +-- Determines which --batch group to build. Normally we want to build all +-- groups, so this arg is intended only for CI, where a single CI job cannot +-- build everything, or it will time out. Hence this is marked 'internal' +-- to hide it from the --help page, as its presence would only confuse. +parseBatchIndex :: Parser (Maybe Int) +parseBatchIndex = + OA.optional $ + OA.option + OA.auto + ( mconcat + [ OA.long "batch-index", + OA.internal, + OA.metavar "NAT" + ] + ) + parseCabalGlobalOpts :: Parser [String] parseCabalGlobalOpts = OA.option @@ -333,72 +355,84 @@ parseColorLogs = bad -> fail $ "Expected one of (detect | on | off), received: " <> bad parseGroupFailFast :: Parser Bool -parseGroupFailFast = - OA.switch - ( mconcat +parseGroupFailFast = mkSwitch opts + where + opts = + mconcat [ OA.long "group-fail-fast", + OA.value False, mkHelp helpTxt ] - ) - where helpTxt = mconcat - [ "If true, the first batch group that fails to completely build stops ", - "clc-stackage." + [ "If on, the first batch group that fails to completely build stops ", + "clc-stackage. Defaults to 'off'." ] -parseNoCabalUpdate :: Parser Bool -parseNoCabalUpdate = - OA.switch - ( mconcat - [ OA.long "no-cabal-update", +parseCabalUpdate :: Parser Bool +parseCabalUpdate = mkSwitch opts + where + opts = + mconcat + [ OA.long "cabal-update", + OA.value True, mkHelpNoLine helpTxt ] - ) - where - helpTxt = "If true, skips the 'cabal update' step." + helpTxt = "Runs 'cabal update' before building. Defaults to 'on'." -parseNoCache :: Parser Bool -parseNoCache = - OA.switch - ( mconcat - [ OA.long "no-cache", +parseCache :: Parser Bool +parseCache = mkSwitch opts + where + opts = + mconcat + [ OA.long "cache", + OA.value True, mkHelp $ mconcat - [ "Disables the cache. Normally, the outcome of a run is saved ", - "to a json cache. This is useful for resuming a run that was ", - "interrupted (e.g. CTRL-C). The next run will fetch the ", - "packages to build from the cache." + [ "Saves the outcome of a run to a json cache, useful for resuming ", + "a run that was interrupted (e.g. CTRL-C). The next run will fetch ", + "the packages to build from the cache. Defaults to 'on'." ] ] - ) -parseNoCleanup :: Parser Bool -parseNoCleanup = - OA.switch - ( mconcat - [ OA.long "no-cleanup", - mkHelp "Will not remove the generated cabal files after exiting." +parseCleanup :: Parser Bool +parseCleanup = mkSwitch opts + where + opts = + mconcat + [ OA.long "cleanup", + OA.value True, + mkHelp "Removes generated files after finishing. Defaults to 'on'." ] - ) parsePackageFailFast :: Parser Bool -parsePackageFailFast = - OA.switch - ( mconcat +parsePackageFailFast = mkSwitch opts + where + opts = + mconcat [ OA.long "package-fail-fast", + OA.value False, mkHelpNoLine helpTxt ] - ) - where helpTxt = mconcat - [ "If true, the first package that fails _within_ a batch group ", + [ "If on, the first package that fails _within_ a batch group ", "will cause the entire group to fail. We then move to the next ", - "group, as normal. The default (off) behavior is equivalent to ", + "group, as normal. The default 'off' behavior is equivalent to ", "cabal's --keep-going)." ] +-- Notice that unlike other on/off switches, this is an actual flag +-- (--print-package-set) vs. an on/off option (--print-package-set (on | off)). +-- +-- We have this exception because this isn't really an on/off switch but +-- rather an alternative command which bypasses the build entirely. This +-- would make more sense using optparse's command syntax, except that would +-- require normal usage to also have some command (e.g. build), which doesn't +-- seem worth it for the normal, happy path. +-- +-- Ideally this would be a command and normal usage would be a "default command", +-- i.e. require no actual command, but optparse has no such notion. parsePrintPackageSet :: Parser Bool parsePrintPackageSet = OA.switch @@ -415,13 +449,18 @@ parsePrintPackageSet = ] parseRetryFailures :: Parser Bool -parseRetryFailures = - OA.switch - ( mconcat +parseRetryFailures = mkSwitch opts + where + opts = + mconcat [ OA.long "retry-failures", - mkHelpNoLine "Retries failures from the cache. Incompatible with --no-cache. " + OA.value False, + mkHelpNoLine $ + mconcat + [ "Retries failures from the cache. Incompatible with '--cache off'. ", + "Defaults to 'off'." + ] ] - ) parseSnapshotPath :: Parser (Maybe OsPath) parseSnapshotPath = @@ -441,7 +480,7 @@ parseSnapshotPath = "https://www.stackage.org//cabal.config i.e. each ", "line should be ' ==' e.g. 'lens ==5.3.4'. Note ", "that the snapshot is still filtered according to ", - "excluded_pkgs.jsonc." + "package_index.jsonc." ] ] ) @@ -561,3 +600,21 @@ cwdPathsCompleter = OAC.mkCompleter $ \word -> do tryIO :: IO a -> IO (Either IOException a) tryIO = try + +-- Makes a switch that takes '(on | off)'. For consistency, this should be +-- preferred for any on/off switch, rather than a normal flag +-- (e.g. --foo (on | off) vs. --foo). +mkSwitch :: Mod OptionFields Bool -> Parser Bool +mkSwitch opts = OA.option readSwitch opts' + where + opts' = + OA.metavar "(on | off)" + <> OA.completeWith ["on", "off"] + <> opts + +readSwitch :: ReadM Bool +readSwitch = + OA.str >>= \case + "off" -> pure False + "on" -> pure True + other -> fail $ "Expected (on | off), received: " ++ other diff --git a/src/CLC/Stackage/Runner/Env.hs b/src/CLC/Stackage/Runner/Env.hs index d5b3104..2def88a 100644 --- a/src/CLC/Stackage/Runner/Env.hs +++ b/src/CLC/Stackage/Runner/Env.hs @@ -28,6 +28,7 @@ import CLC.Stackage.Builder.Env ), ) import CLC.Stackage.Builder.Env qualified as Builder.Env +import CLC.Stackage.Parser (PackageSet (packageList)) import CLC.Stackage.Parser qualified as Parser import CLC.Stackage.Runner.Args ( Args (snapshotPath), @@ -43,10 +44,10 @@ import CLC.Stackage.Runner.Report qualified as Report import CLC.Stackage.Utils.Exception qualified as Ex import CLC.Stackage.Utils.IO qualified as IO import CLC.Stackage.Utils.Logging qualified as Logging -import CLC.Stackage.Utils.Package (Package (MkPackage, name, version)) +import CLC.Stackage.Utils.Package (Package) import CLC.Stackage.Utils.Paths qualified as Paths import Control.Exception (throwIO) -import Control.Monad (join, unless, when) +import Control.Monad (join, when) import Data.Bool (Bool (False, True), not) import Data.Foldable (Foldable (foldl')) import Data.IORef (newIORef, readIORef) @@ -61,26 +62,26 @@ import System.Directory.OsPath qualified as Dir import System.Exit (ExitCode (ExitSuccess)) import System.OsPath (osp) import System.OsPath qualified as OsP -import Prelude (IO, Monad ((>>=)), mconcat, pure, show, ($), (.), (<$>), (<>)) +import Prelude (IO, Monad ((>>=)), mconcat, pure, show, ($), (.), (<>)) -- | Args used for building all packages. data RunnerEnv = MkRunnerEnv { -- | Environment used in building. buildEnv :: BuildEnv, + -- | Enables the 'cabal update' step. + cabalUpdate :: Bool, -- | Status from previous run. cache :: Maybe Results, + -- | Enables the cache, which saves the outcome of a run in a json file. + -- The cache is used for resuming a run that was interrupted. + cacheEnabled :: Bool, + -- | If disabled, we do not revert the cabal file at the end (i.e. we + -- leave the last attempted build). + cleanup :: Bool, -- | The complete package set from stackage. This is used to write the -- cabal.project.local's constraint section, to ensure we always use the -- same transitive dependencies. - completePackageSet :: [Package], - -- | Disables the 'cabal update' step. - noCabalUpdate :: Bool, - -- | Disables the cache, which otherwise saves the outcome of a run in a - -- json file. The cache is used for resuming a run that was interrupted. - noCache :: Bool, - -- | If we do not revert the cabal file at the end (i.e. we leave the - -- last attempted build). - noCleanup :: Bool, + completePackageSet :: PackageSet, -- | Whether to retry packages that failed. retryFailures :: Bool, -- | Start time. @@ -136,18 +137,17 @@ setup hLoggerRaw modifyPackages = do failuresRef <- newIORef Set.empty cache <- - if cliArgs.noCache - then pure Nothing - else Report.readCache hLogger + if cliArgs.cache + then Report.readCache hLogger + else pure Nothing -- (entire set, packages to build) (completePackageSet, pkgsList) <- case cache of Nothing -> do -- if no cache exists, query stackage - pkgsResponses <- Parser.getPackageList hLogger cliArgs.snapshotPath - let completePackageSet = responseToPkgs <$> pkgsResponses - pkgs = modifyPackages completePackageSet - pure (completePackageSet, pkgs) + packageSet <- Parser.getPackageList hLogger cliArgs.snapshotPath + let pkgs = modifyPackages packageSet.packageList + pure (packageSet, pkgs) Just oldResults -> do -- cache exists, use it rather than stackage oldFailures <- @@ -164,7 +164,11 @@ setup hLoggerRaw modifyPackages = do untested = oldResults.untested toTest = Set.union untested oldFailures - pure (Set.toList completePackageSet, Set.toList toTest) + -- Even though we read the packages from the cache, we still want to + -- get the packageSet, for writing the cabal.project.local constraints. + packageSet <- Parser.packageListToSet hLogger (Set.toList completePackageSet) + + pure (packageSet, Set.toList toTest) packagesToBuild <- case pkgsList of (p : ps) -> pure (p :| ps) @@ -181,6 +185,7 @@ setup hLoggerRaw modifyPackages = do buildEnv = MkBuildEnv { batch = cliArgs.batch, + batchIndex = cliArgs.batchIndex, buildArgs, cabalPath, groupFailFast = cliArgs.groupFailFast, @@ -199,26 +204,22 @@ setup hLoggerRaw modifyPackages = do { buildEnv, cache, completePackageSet, - noCabalUpdate = cliArgs.noCabalUpdate, - noCache = cliArgs.noCache, - noCleanup = cliArgs.noCleanup, + cabalUpdate = cliArgs.cabalUpdate, + cacheEnabled = cliArgs.cache, + cleanup = cliArgs.cleanup, retryFailures = cliArgs.retryFailures, startTime } - where - responseToPkgs p = - MkPackage - { name = p.name, - version = p.version - } -- | Prints summary and writes results to disk. teardown :: RunnerEnv -> IO () teardown env = do endTime <- env.buildEnv.hLogger.getLocalTime - unless env.noCleanup $ do - Dir.removeFile Paths.generatedCabalPath - Dir.removeFile Paths.generatedCabalProjectLocalPath + when env.cleanup $ do + -- removePathForcibly as these might not exist e.g. if the we did not + -- build anything. + Dir.removePathForcibly Paths.generatedCabalPath + Dir.removePathForcibly Paths.generatedCabalProjectLocalPath results <- getResults env.buildEnv let report = @@ -227,7 +228,7 @@ teardown env = do (Logging.formatLocalTime env.startTime) (Logging.formatLocalTime endTime) - unless env.noCache (updateCache env results) + when env.cacheEnabled (updateCache env results) Report.saveReport report diff --git a/test/functional/Main.hs b/test/functional/Main.hs index 7ddb8a1..697fc5a 100644 --- a/test/functional/Main.hs +++ b/test/functional/Main.hs @@ -161,10 +161,13 @@ runGolden getNoCleanup params = -- While NOTE: [Skipping cleanup] will prevent the test cleanup from running, -- the clc-stackage also performs a cleanup. Thus if no cleanup is desired - -- (NO_CLEANUP is set), we also need to pass the --no-cleanup arg to the + -- (NO_CLEANUP is set), we also need to pass the '--cleanup off' arg to the -- exe. noCleanup <- getNoCleanup - let noCleanupArgs = ["--no-cleanup" | noCleanup] + let noCleanupArgs = + if noCleanup + then [] + else ["--cleanup", "off"] finalArgs = args' ++ noCleanupArgs logs <- withArgs finalArgs params.runner @@ -177,13 +180,18 @@ runGolden getNoCleanup params = skipLog = BS.isInfixOf "PATH and Stackage ghc" -- Strip non-determinism from logs (e.g. version numbers, snapshots). + -- At most one of these should match. If none do, we return the original + -- string. massageLogs bs = fromMaybe bs $ - asum $ fmap ($ bs) - [ fixGhcStr, - fixSnapshotStr, - fixNumPkgs - ] + asum $ + fmap + ($ bs) + [ fixGhcStr, + fixSnapshotStr, + fixNumPkgs, + fixLibs + ] where fixGhcStr b = do (pre, r1) <- Parser.Utils.stripInfix "ghc: " b @@ -201,11 +209,47 @@ runGolden getNoCleanup params = let (pre, _num) = BS.breakEnd (not . Parser.Utils.isNum) r1 pure $ pre <> " packages" <> post + -- Idea: For a given bytestring, try to find an expected lib string + -- e.g. 'aeson'. If we find it, we place the version number with + -- '', then recursively run on the rest of the string. + fixLibs :: ByteString -> Maybe ByteString + fixLibs b = do + (p1, r1) <- fixLib b + pure $ p1 <> fromMaybe r1 (fixLibs r1) + where + fixLib :: ByteString -> Maybe (ByteString, ByteString) + fixLib c = asum $ fmap (tryLib c) libs + + libs = + [ "aeson", + "cborg", + "clock", + "extra", + "kan-extensions", + "mtl", + "optics-core", + "profunctors", + "servant" + ] + + -- E.g. tryLib "abc lib-1.2.3 def" "lib" + -- Just ("abc lib-"," def") + tryLib :: ByteString -> ByteString -> Maybe (ByteString, ByteString) + tryLib c lib = do + (pre, r1) <- Parser.Utils.stripInfix lib c + let (_vers, rest) = BS.break isDelim r1 + pure (pre <> lib <> "-", rest) + where + isDelim d = + d == Parser.Utils.commaW8 + || d == Parser.Utils.spaceW8 + -- test w/ color off since CI can't handle it, apparently args' = "--color-logs" : "off" - : "--no-cabal-update" + : "--cabal-update" + : "off" : params.args baseTestPath = diff --git a/test/functional/goldens/testSmallBatch_linux.golden b/test/functional/goldens/testSmallBatch_linux.golden index 0927d98..4f511d1 100644 --- a/test/functional/goldens/testSmallBatch_linux.golden +++ b/test/functional/goldens/testSmallBatch_linux.golden @@ -4,9 +4,9 @@ [2020-05-31 12:00:00][Info] Stackage ghc: [2020-05-31 12:00:00][Info] Filtered to packages (linux). [2020-05-31 12:00:00][Info] Starting build(s) -[2020-05-31 12:00:00][Success] 3: cborg-0.2.10.0, clock-0.8.4 -[2020-05-31 12:00:00][Success] 2: extra-1.8.1, optics-core-0.4.2 -[2020-05-31 12:00:00][Success] 1: profunctors-5.6.3 +[2020-05-31 12:00:00][Success] 3: cborg-, clock- +[2020-05-31 12:00:00][Success] 2: extra-, optics-core- +[2020-05-31 12:00:00][Success] 1: profunctors- - Successes: 5 (100%) diff --git a/test/functional/goldens/testSmallBatch_osx.golden b/test/functional/goldens/testSmallBatch_osx.golden index 4010f18..cc88dac 100644 --- a/test/functional/goldens/testSmallBatch_osx.golden +++ b/test/functional/goldens/testSmallBatch_osx.golden @@ -4,9 +4,9 @@ [2020-05-31 12:00:00][Info] Stackage ghc: [2020-05-31 12:00:00][Info] Filtered to packages (osx). [2020-05-31 12:00:00][Info] Starting build(s) -[2020-05-31 12:00:00][Success] 3: cborg-0.2.10.0, clock-0.8.4 -[2020-05-31 12:00:00][Success] 2: extra-1.8.1, optics-core-0.4.2 -[2020-05-31 12:00:00][Success] 1: profunctors-5.6.3 +[2020-05-31 12:00:00][Success] 3: cborg-, clock- +[2020-05-31 12:00:00][Success] 2: extra-, optics-core- +[2020-05-31 12:00:00][Success] 1: profunctors- - Successes: 5 (100%) diff --git a/test/functional/goldens/testSmallBatch_windows.golden b/test/functional/goldens/testSmallBatch_windows.golden index 9156c72..7c6be8f 100644 --- a/test/functional/goldens/testSmallBatch_windows.golden +++ b/test/functional/goldens/testSmallBatch_windows.golden @@ -4,9 +4,9 @@ [2020-05-31 12:00:00][Info] Stackage ghc: [2020-05-31 12:00:00][Info] Filtered to packages (windows). [2020-05-31 12:00:00][Info] Starting build(s) -[2020-05-31 12:00:00][Success] 3: cborg-0.2.10.0, clock-0.8.4 -[2020-05-31 12:00:00][Success] 2: extra-1.8.1, optics-core-0.4.2 -[2020-05-31 12:00:00][Success] 1: profunctors-5.6.3 +[2020-05-31 12:00:00][Success] 3: cborg-, clock- +[2020-05-31 12:00:00][Success] 2: extra-, optics-core- +[2020-05-31 12:00:00][Success] 1: profunctors- - Successes: 5 (100%) diff --git a/test/functional/goldens/testSmallSnapshotPath_linux.golden b/test/functional/goldens/testSmallSnapshotPath_linux.golden index 6fea1a1..5e14c0b 100644 --- a/test/functional/goldens/testSmallSnapshotPath_linux.golden +++ b/test/functional/goldens/testSmallSnapshotPath_linux.golden @@ -5,7 +5,7 @@ [2020-05-31 12:00:00][Info] Stackage ghc: [2020-05-31 12:00:00][Info] Filtered to packages (linux). [2020-05-31 12:00:00][Info] Starting build(s) -[2020-05-31 12:00:00][Success] 1: cborg-0.2.10.0, clock-0.8.4, extra-1.8, op... +[2020-05-31 12:00:00][Success] 1: cborg-, clock-, extra-, op... - Successes: 5 (100%) diff --git a/test/functional/goldens/testSmallSnapshotPath_osx.golden b/test/functional/goldens/testSmallSnapshotPath_osx.golden index e360336..9eb342b 100644 --- a/test/functional/goldens/testSmallSnapshotPath_osx.golden +++ b/test/functional/goldens/testSmallSnapshotPath_osx.golden @@ -5,7 +5,7 @@ [2020-05-31 12:00:00][Info] Stackage ghc: [2020-05-31 12:00:00][Info] Filtered to packages (osx). [2020-05-31 12:00:00][Info] Starting build(s) -[2020-05-31 12:00:00][Success] 1: cborg-0.2.10.0, clock-0.8.4, extra-1.8, op... +[2020-05-31 12:00:00][Success] 1: cborg-, clock-, extra-, op... - Successes: 5 (100%) diff --git a/test/functional/goldens/testSmallSnapshotPath_windows.golden b/test/functional/goldens/testSmallSnapshotPath_windows.golden index 46d4a5c..224bbcf 100644 --- a/test/functional/goldens/testSmallSnapshotPath_windows.golden +++ b/test/functional/goldens/testSmallSnapshotPath_windows.golden @@ -5,7 +5,7 @@ [2020-05-31 12:00:00][Info] Stackage ghc: [2020-05-31 12:00:00][Info] Filtered to packages (windows). [2020-05-31 12:00:00][Info] Starting build(s) -[2020-05-31 12:00:00][Success] 1: cborg-0.2.10.0, clock-0.8.4, extra-1.8, op... +[2020-05-31 12:00:00][Success] 1: cborg-, clock-, extra-, op... - Successes: 5 (100%) diff --git a/test/functional/goldens/testSmall_linux.golden b/test/functional/goldens/testSmall_linux.golden index 9dddebd..d12cdef 100644 --- a/test/functional/goldens/testSmall_linux.golden +++ b/test/functional/goldens/testSmall_linux.golden @@ -4,7 +4,7 @@ [2020-05-31 12:00:00][Info] Stackage ghc: [2020-05-31 12:00:00][Info] Filtered to packages (linux). [2020-05-31 12:00:00][Info] Starting build(s) -[2020-05-31 12:00:00][Success] 1: cborg-0.2.10.0, clock-0.8.4, extra-1.8.1, ... +[2020-05-31 12:00:00][Success] 1: cborg-, clock-, extra-, ... - Successes: 5 (100%) diff --git a/test/functional/goldens/testSmall_osx.golden b/test/functional/goldens/testSmall_osx.golden index 5d67dff..74548e0 100644 --- a/test/functional/goldens/testSmall_osx.golden +++ b/test/functional/goldens/testSmall_osx.golden @@ -4,7 +4,7 @@ [2020-05-31 12:00:00][Info] Stackage ghc: [2020-05-31 12:00:00][Info] Filtered to packages (osx). [2020-05-31 12:00:00][Info] Starting build(s) -[2020-05-31 12:00:00][Success] 1: cborg-0.2.10.0, clock-0.8.4, extra-1.8.1, ... +[2020-05-31 12:00:00][Success] 1: cborg-, clock-, extra-, ... - Successes: 5 (100%) diff --git a/test/functional/goldens/testSmall_windows.golden b/test/functional/goldens/testSmall_windows.golden index 6c649cc..d35d2f1 100644 --- a/test/functional/goldens/testSmall_windows.golden +++ b/test/functional/goldens/testSmall_windows.golden @@ -4,7 +4,7 @@ [2020-05-31 12:00:00][Info] Stackage ghc: [2020-05-31 12:00:00][Info] Filtered to packages (windows). [2020-05-31 12:00:00][Info] Starting build(s) -[2020-05-31 12:00:00][Success] 1: cborg-0.2.10.0, clock-0.8.4, extra-1.8.1, ... +[2020-05-31 12:00:00][Success] 1: cborg-, clock-, extra-, ... - Successes: 5 (100%) diff --git a/test/unit/Unit/Prelude.hs b/test/unit/Unit/Prelude.hs index 2d7f94d..3f7ba77 100644 --- a/test/unit/Unit/Prelude.hs +++ b/test/unit/Unit/Prelude.hs @@ -9,6 +9,7 @@ import CLC.Stackage.Builder.Env ( BuildEnv ( MkBuildEnv, batch, + batchIndex, buildArgs, cabalPath, groupFailFast, @@ -19,15 +20,23 @@ import CLC.Stackage.Builder.Env ), Progress (MkProgress, failuresRef, successesRef), ) +import CLC.Stackage.Parser + ( PackageSet + ( MkPackageSet, + extraPins, + packageList, + unpinned + ), + ) import CLC.Stackage.Runner.Env ( RunnerEnv ( MkRunnerEnv, buildEnv, + cabalUpdate, cache, + cacheEnabled, + cleanup, completePackageSet, - noCabalUpdate, - noCache, - noCleanup, retryFailures, startTime ), @@ -50,10 +59,15 @@ mkRunnerEnv = do MkRunnerEnv { buildEnv, cache = Nothing, - completePackageSet = NE.toList buildEnv.packagesToBuild, - noCabalUpdate = True, - noCache = False, - noCleanup = False, + completePackageSet = + MkPackageSet + { extraPins = [], + packageList = NE.toList buildEnv.packagesToBuild, + unpinned = [] + }, + cabalUpdate = False, + cacheEnabled = True, + cleanup = True, retryFailures = False, startTime = mkLocalTime } @@ -66,6 +80,7 @@ mkBuildEnv = do pure $ MkBuildEnv { batch = Nothing, + batchIndex = Nothing, buildArgs = [], cabalPath = "cabal", groupFailFast = False,