From 10c545337b87744612639596cd2b38a1716df7c0 Mon Sep 17 00:00:00 2001 From: Tommy Bidne Date: Mon, 6 Apr 2026 10:05:04 +1200 Subject: [PATCH 1/5] Clarify adding custom GHC to PATH --- README.md | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index a9c6def..c4d8824 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,15 @@ The procedure is as follows: with-compiler: /home/ghc/_build/stage1/bin/ghc ``` -6. Run `clc-stackage` and wait for a long time. See [below](#the-clc-stackage-exe) for more details. +6. Add your custom GHC to the PATH e.g. + + ``` + export PATH=/home/ghc/_build/stage1/bin/:$PATH + ``` + + Nix users can uncomment (and modify) this line in the `flake.nix`. + +7. Run `clc-stackage` and wait for a long time. See [below](#the-clc-stackage-exe) for more details. * On a recent Macbook Air it takes around 12 hours, YMMV. * You can interrupt `cabal` at any time and rerun again later. @@ -51,14 +59,14 @@ The procedure is as follows: $ watch -n 10 "grep -Eo 'Completed|^ -' output/logs/current-build/stdout.log | sort -r | uniq -c | awk '{print \$1}'" ``` -7. If any packages fail to compile: +8. If any packages fail to compile: * copy them locally using `cabal unpack`, * patch to confirm with your proposal, * link them from `packages` section of `cabal.project`, * return to Step 6. -8. When everything finally builds, get back to CLC with a list of packages affected and patches required. +9. When everything finally builds, get back to CLC with a list of packages affected and patches required. ### Troubleshooting @@ -189,13 +197,3 @@ For Linux based systems, there's a provided `flake.nix` and `shell.nix` to get a with an approximation of the required dependencies (cabal itself, C libs) to build `clc-stackage`. Note that it is not actively maintained, so it may require some tweaking to get working, and conversely, it may have some redundant dependencies. - -## Misc - -* Your custom GHC will need to be on the PATH to build the `stack` library e.g. - - ``` - export PATH=/home/ghc/_build/stage1/bin/:$PATH - ``` - - Nix users can uncomment (and modify) this line in the `flake.nix`. From c02038ba711d258b4d18ff8df0efe8b2206ae4be Mon Sep 17 00:00:00 2001 From: Tommy Bidne Date: Tue, 7 Apr 2026 20:18:07 +1200 Subject: [PATCH 2/5] Improve functional test non-determinism --- dev.md | 6 +-- src/CLC/Stackage/Parser/Utils.hs | 6 ++- test/functional/Main.hs | 50 +++++++++++++++++-- .../goldens/testSmallBatch_linux.golden | 6 +-- .../goldens/testSmallBatch_osx.golden | 6 +-- .../goldens/testSmallBatch_windows.golden | 6 +-- .../testSmallSnapshotPath_linux.golden | 2 +- .../goldens/testSmallSnapshotPath_osx.golden | 2 +- .../testSmallSnapshotPath_windows.golden | 2 +- .../functional/goldens/testSmall_linux.golden | 2 +- test/functional/goldens/testSmall_osx.golden | 2 +- .../goldens/testSmall_windows.golden | 2 +- 12 files changed, 67 insertions(+), 25 deletions(-) diff --git a/dev.md b/dev.md index 186cfbd..48d9c63 100644 --- a/dev.md +++ b/dev.md @@ -81,9 +81,7 @@ The executable that actually runs. This is a very thin wrapper over `runner`, wh 2. Update `ghc-version` in [.github/workflows/ci.yaml](.github/workflows/ci.yaml). -3. Update functional tests as needed i.e. exact package versions in `*golden` and `test/functional/snapshot.txt`. - -4. Optional: Update nix: +3. Optional: Update nix: - Inputs (`nix flake update`). - GHC: Update the `compiler = pkgs.haskell.packages.ghc;` line. @@ -91,7 +89,7 @@ The executable that actually runs. This is a very thin wrapper over `runner`, wh This job builds everything with `--dry-run`, so its success is a useful proxy for `clc-stackage`'s health. In other words, if the nix job fails, there is almost certainly a general issue (i.e. either a package should be excluded or new system dep is required), but if it succeeds, the package set is in pretty good shape (there may still be sporadic issues e.g. a package does not properly declare its system dependencies at config time). -5. Optional: Update `clc-stackage.cabal`'s dependencies (i.e. `cabal outdated`). +4. Optional: Update `clc-stackage.cabal`'s dependencies (i.e. `cabal outdated`). ### Verifying snapshot diff --git a/src/CLC/Stackage/Parser/Utils.hs b/src/CLC/Stackage/Parser/Utils.hs index c19ef04..c0ab84e 100644 --- a/src/CLC/Stackage/Parser/Utils.hs +++ b/src/CLC/Stackage/Parser/Utils.hs @@ -5,6 +5,7 @@ module CLC.Stackage.Parser.Utils -- * Misc isNum, + commaW8, spaceW8, ) where @@ -102,8 +103,11 @@ fslashW8 = i2w8 $ Ch.ord '/' spaceW8 :: Word8 spaceW8 = i2w8 $ Ch.ord ' ' +commaW8 :: Word8 +commaW8 = i2w8 $ Ch.ord ',' + isNum :: Word8 -> Bool -isNum w = w >= (i2w8 $ Ch.ord '0') && w <= (i2w8 $ Ch.ord '9') +isNum w = w >= i2w8 (Ch.ord '0') && w <= i2w8 (Ch.ord '9') i2w8 :: Int -> Word8 i2w8 = fromIntegral diff --git a/test/functional/Main.hs b/test/functional/Main.hs index 7ddb8a1..95d685e 100644 --- a/test/functional/Main.hs +++ b/test/functional/Main.hs @@ -177,13 +177,18 @@ runGolden getNoCleanup params = skipLog = BS.isInfixOf "PATH and Stackage ghc" -- Strip non-determinism from logs (e.g. version numbers, snapshots). + -- At most one of these should match. If none do, we return the original + -- string. massageLogs bs = fromMaybe bs $ - asum $ fmap ($ bs) - [ fixGhcStr, - fixSnapshotStr, - fixNumPkgs - ] + asum $ + fmap + ($ bs) + [ fixGhcStr, + fixSnapshotStr, + fixNumPkgs, + fixLibs + ] where fixGhcStr b = do (pre, r1) <- Parser.Utils.stripInfix "ghc: " b @@ -201,6 +206,41 @@ runGolden getNoCleanup params = let (pre, _num) = BS.breakEnd (not . Parser.Utils.isNum) r1 pure $ pre <> " packages" <> post + -- Idea: For a given bytestring, try to find an expected lib string + -- e.g. 'aeson'. If we find it, we place the version number with + -- '', then recursively run on the rest of the string. + fixLibs :: ByteString -> Maybe ByteString + fixLibs b = do + (p1, r1) <- fixLib b + pure $ p1 <> fromMaybe r1 (fixLibs r1) + where + fixLib :: ByteString -> Maybe (ByteString, ByteString) + fixLib c = asum $ fmap (tryLib c) libs + + libs = + [ "aeson", + "cborg", + "clock", + "extra", + "kan-extensions", + "mtl", + "optics-core", + "profunctors", + "servant" + ] + + -- E.g. tryLib "abc lib-1.2.3 def" "lib" + -- Just ("abc lib-"," def") + tryLib :: ByteString -> ByteString -> Maybe (ByteString, ByteString) + tryLib c lib = do + (pre, r1) <- Parser.Utils.stripInfix lib c + let (_vers, rest) = BS.break isDelim r1 + pure (pre <> lib <> "-", rest) + where + isDelim d = + d == Parser.Utils.commaW8 + || d == Parser.Utils.spaceW8 + -- test w/ color off since CI can't handle it, apparently args' = "--color-logs" diff --git a/test/functional/goldens/testSmallBatch_linux.golden b/test/functional/goldens/testSmallBatch_linux.golden index 0927d98..4f511d1 100644 --- a/test/functional/goldens/testSmallBatch_linux.golden +++ b/test/functional/goldens/testSmallBatch_linux.golden @@ -4,9 +4,9 @@ [2020-05-31 12:00:00][Info] Stackage ghc: [2020-05-31 12:00:00][Info] Filtered to packages (linux). [2020-05-31 12:00:00][Info] Starting build(s) -[2020-05-31 12:00:00][Success] 3: cborg-0.2.10.0, clock-0.8.4 -[2020-05-31 12:00:00][Success] 2: extra-1.8.1, optics-core-0.4.2 -[2020-05-31 12:00:00][Success] 1: profunctors-5.6.3 +[2020-05-31 12:00:00][Success] 3: cborg-, clock- +[2020-05-31 12:00:00][Success] 2: extra-, optics-core- +[2020-05-31 12:00:00][Success] 1: profunctors- - Successes: 5 (100%) diff --git a/test/functional/goldens/testSmallBatch_osx.golden b/test/functional/goldens/testSmallBatch_osx.golden index 4010f18..cc88dac 100644 --- a/test/functional/goldens/testSmallBatch_osx.golden +++ b/test/functional/goldens/testSmallBatch_osx.golden @@ -4,9 +4,9 @@ [2020-05-31 12:00:00][Info] Stackage ghc: [2020-05-31 12:00:00][Info] Filtered to packages (osx). [2020-05-31 12:00:00][Info] Starting build(s) -[2020-05-31 12:00:00][Success] 3: cborg-0.2.10.0, clock-0.8.4 -[2020-05-31 12:00:00][Success] 2: extra-1.8.1, optics-core-0.4.2 -[2020-05-31 12:00:00][Success] 1: profunctors-5.6.3 +[2020-05-31 12:00:00][Success] 3: cborg-, clock- +[2020-05-31 12:00:00][Success] 2: extra-, optics-core- +[2020-05-31 12:00:00][Success] 1: profunctors- - Successes: 5 (100%) diff --git a/test/functional/goldens/testSmallBatch_windows.golden b/test/functional/goldens/testSmallBatch_windows.golden index 9156c72..7c6be8f 100644 --- a/test/functional/goldens/testSmallBatch_windows.golden +++ b/test/functional/goldens/testSmallBatch_windows.golden @@ -4,9 +4,9 @@ [2020-05-31 12:00:00][Info] Stackage ghc: [2020-05-31 12:00:00][Info] Filtered to packages (windows). [2020-05-31 12:00:00][Info] Starting build(s) -[2020-05-31 12:00:00][Success] 3: cborg-0.2.10.0, clock-0.8.4 -[2020-05-31 12:00:00][Success] 2: extra-1.8.1, optics-core-0.4.2 -[2020-05-31 12:00:00][Success] 1: profunctors-5.6.3 +[2020-05-31 12:00:00][Success] 3: cborg-, clock- +[2020-05-31 12:00:00][Success] 2: extra-, optics-core- +[2020-05-31 12:00:00][Success] 1: profunctors- - Successes: 5 (100%) diff --git a/test/functional/goldens/testSmallSnapshotPath_linux.golden b/test/functional/goldens/testSmallSnapshotPath_linux.golden index 6fea1a1..5e14c0b 100644 --- a/test/functional/goldens/testSmallSnapshotPath_linux.golden +++ b/test/functional/goldens/testSmallSnapshotPath_linux.golden @@ -5,7 +5,7 @@ [2020-05-31 12:00:00][Info] Stackage ghc: [2020-05-31 12:00:00][Info] Filtered to packages (linux). [2020-05-31 12:00:00][Info] Starting build(s) -[2020-05-31 12:00:00][Success] 1: cborg-0.2.10.0, clock-0.8.4, extra-1.8, op... +[2020-05-31 12:00:00][Success] 1: cborg-, clock-, extra-, op... - Successes: 5 (100%) diff --git a/test/functional/goldens/testSmallSnapshotPath_osx.golden b/test/functional/goldens/testSmallSnapshotPath_osx.golden index e360336..9eb342b 100644 --- a/test/functional/goldens/testSmallSnapshotPath_osx.golden +++ b/test/functional/goldens/testSmallSnapshotPath_osx.golden @@ -5,7 +5,7 @@ [2020-05-31 12:00:00][Info] Stackage ghc: [2020-05-31 12:00:00][Info] Filtered to packages (osx). [2020-05-31 12:00:00][Info] Starting build(s) -[2020-05-31 12:00:00][Success] 1: cborg-0.2.10.0, clock-0.8.4, extra-1.8, op... +[2020-05-31 12:00:00][Success] 1: cborg-, clock-, extra-, op... - Successes: 5 (100%) diff --git a/test/functional/goldens/testSmallSnapshotPath_windows.golden b/test/functional/goldens/testSmallSnapshotPath_windows.golden index 46d4a5c..224bbcf 100644 --- a/test/functional/goldens/testSmallSnapshotPath_windows.golden +++ b/test/functional/goldens/testSmallSnapshotPath_windows.golden @@ -5,7 +5,7 @@ [2020-05-31 12:00:00][Info] Stackage ghc: [2020-05-31 12:00:00][Info] Filtered to packages (windows). [2020-05-31 12:00:00][Info] Starting build(s) -[2020-05-31 12:00:00][Success] 1: cborg-0.2.10.0, clock-0.8.4, extra-1.8, op... +[2020-05-31 12:00:00][Success] 1: cborg-, clock-, extra-, op... - Successes: 5 (100%) diff --git a/test/functional/goldens/testSmall_linux.golden b/test/functional/goldens/testSmall_linux.golden index 9dddebd..d12cdef 100644 --- a/test/functional/goldens/testSmall_linux.golden +++ b/test/functional/goldens/testSmall_linux.golden @@ -4,7 +4,7 @@ [2020-05-31 12:00:00][Info] Stackage ghc: [2020-05-31 12:00:00][Info] Filtered to packages (linux). [2020-05-31 12:00:00][Info] Starting build(s) -[2020-05-31 12:00:00][Success] 1: cborg-0.2.10.0, clock-0.8.4, extra-1.8.1, ... +[2020-05-31 12:00:00][Success] 1: cborg-, clock-, extra-, ... - Successes: 5 (100%) diff --git a/test/functional/goldens/testSmall_osx.golden b/test/functional/goldens/testSmall_osx.golden index 5d67dff..74548e0 100644 --- a/test/functional/goldens/testSmall_osx.golden +++ b/test/functional/goldens/testSmall_osx.golden @@ -4,7 +4,7 @@ [2020-05-31 12:00:00][Info] Stackage ghc: [2020-05-31 12:00:00][Info] Filtered to packages (osx). [2020-05-31 12:00:00][Info] Starting build(s) -[2020-05-31 12:00:00][Success] 1: cborg-0.2.10.0, clock-0.8.4, extra-1.8.1, ... +[2020-05-31 12:00:00][Success] 1: cborg-, clock-, extra-, ... - Successes: 5 (100%) diff --git a/test/functional/goldens/testSmall_windows.golden b/test/functional/goldens/testSmall_windows.golden index 6c649cc..d35d2f1 100644 --- a/test/functional/goldens/testSmall_windows.golden +++ b/test/functional/goldens/testSmall_windows.golden @@ -4,7 +4,7 @@ [2020-05-31 12:00:00][Info] Stackage ghc: [2020-05-31 12:00:00][Info] Filtered to packages (windows). [2020-05-31 12:00:00][Info] Starting build(s) -[2020-05-31 12:00:00][Success] 1: cborg-0.2.10.0, clock-0.8.4, extra-1.8.1, ... +[2020-05-31 12:00:00][Success] 1: cborg-, clock-, extra-, ... - Successes: 5 (100%) From 90282ce5b3f40b67504ea29068dffe166554b60b Mon Sep 17 00:00:00 2001 From: Tommy Bidne Date: Tue, 7 Apr 2026 21:04:45 +1200 Subject: [PATCH 3/5] Switch --no-X flags to (on/off) options For consistency (other options with 'off') and simplicity (avoid double negatives), switch --no-X flags to on/off switches. For instance, instead of '--no-cache', we have '--cache (on | off)' that defaults to 'on'. Other flags get the same treatment e.g. '--group-fail-fast (on | off)' rather than '--group-fail-fast'. The one exception is '--print-package-set' -- which remains a flag -- though that is because it is really an alternative command, not a switch. --- src/CLC/Stackage/Runner.hs | 4 +- src/CLC/Stackage/Runner/Args.hs | 157 +++++++++++++++++++------------- src/CLC/Stackage/Runner/Env.hs | 34 +++---- test/functional/Main.hs | 10 +- test/unit/Unit/Prelude.hs | 12 +-- 5 files changed, 128 insertions(+), 89 deletions(-) diff --git a/src/CLC/Stackage/Runner.hs b/src/CLC/Stackage/Runner.hs index 1ade3d5..804e22d 100644 --- a/src/CLC/Stackage/Runner.hs +++ b/src/CLC/Stackage/Runner.hs @@ -17,7 +17,7 @@ import CLC.Stackage.Runner.Env qualified as Env import CLC.Stackage.Utils.Logging qualified as Logging import CLC.Stackage.Utils.Package (Package) import Control.Exception (bracket, throwIO) -import Control.Monad (unless, when) +import Control.Monad (when) import Data.Foldable (for_) import Data.IORef (readIORef) import System.Exit (ExitCode (ExitFailure)) @@ -48,7 +48,7 @@ runModifyPackages hLogger modifyPackages = withHiddenInput $ do -- write the entire package set to the cabal.project.local's constraints Writer.writeCabalProjectLocal env.completePackageSet - unless env.noCabalUpdate $ Builder.cabalUpdate buildEnv + when env.cabalUpdate $ Builder.cabalUpdate buildEnv Logging.putTimeInfoStr buildEnv.hLogger "Starting build(s)" diff --git a/src/CLC/Stackage/Runner/Args.hs b/src/CLC/Stackage/Runner/Args.hs index 431a25d..c1cb23e 100644 --- a/src/CLC/Stackage/Runner/Args.hs +++ b/src/CLC/Stackage/Runner/Args.hs @@ -64,19 +64,19 @@ data Args = MkArgs cabalOpts :: [String], -- | Optional path to cabal executable. cabalPath :: Maybe OsPath, + -- | If true, the 'cabal update' step is run. + cabalUpdate :: Bool, + -- | Enables the cache, which saves the outcome of a run in a json file. + -- The cache is used for resuming a run that was interrupted. + cache :: Bool, + -- | If true, leaves the last generated cabal files. + cleanup :: Bool, -- | Determines if we color the logs. If 'Nothing', attempts to detect -- if colors are supported. colorLogs :: ColorLogs, -- | If true, the first group that fails to completely build stops -- clc-stackage. groupFailFast :: Bool, - -- | If true, the 'cabal update' step is skipped. - noCabalUpdate :: Bool, - -- | Disables the cache, which otherwise saves the outcome of a run in a - -- json file. The cache is used for resuming a run that was interrupted. - noCache :: Bool, - -- | If true, leaves the last generated cabal files. - noCleanup :: Bool, -- | If true, the first package that fails _within_ a package group will -- cause the entire group to fail. packageFailFast :: Bool, @@ -188,11 +188,11 @@ getArgs = OA.execParser parserInfoArgs parseCliArgs :: Parser Args parseCliArgs = ( do - ~(cabalGlobalOpts, cabalOpts, cabalPath, noCabalUpdate) <- parseCabalGroup - ~(noCache, retryFailures) <- parseCacheGroup + ~(cabalGlobalOpts, cabalOpts, cabalPath, cabalUpdate) <- parseCabalGroup + ~(cache, retryFailures) <- parseCacheGroup ~(groupFailFast, packageFailFast) <- parseFailuresGroup ~(batch, printPackageSet, snapshotPath) <- parseMiscGroup - ~(colorLogs, noCleanup, writeLogs) <- parseOutputGroup + ~(cleanup, colorLogs, writeLogs) <- parseOutputGroup pure $ MkArgs @@ -200,11 +200,11 @@ parseCliArgs = cabalGlobalOpts, cabalOpts, cabalPath, + cabalUpdate, + cache, + cleanup, colorLogs, groupFailFast, - noCabalUpdate, - noCache, - noCleanup, packageFailFast, printPackageSet, retryFailures, @@ -220,12 +220,12 @@ parseCliArgs = <$> parseCabalGlobalOpts <*> parseCabalOpts <*> parseCabalPath - <*> parseNoCabalUpdate + <*> parseCabalUpdate parseCacheGroup = OA.parserOptionGroup "Cache options:" $ (,) - <$> parseNoCache + <$> parseCache <*> parseRetryFailures parseFailuresGroup = @@ -244,8 +244,8 @@ parseCliArgs = parseOutputGroup = OA.parserOptionGroup "Output options:" $ (,,) - <$> parseColorLogs - <*> parseNoCleanup + <$> parseCleanup + <*> parseColorLogs <*> parseWriteLogs parseBatch :: Parser (Maybe Int) @@ -333,72 +333,84 @@ parseColorLogs = bad -> fail $ "Expected one of (detect | on | off), received: " <> bad parseGroupFailFast :: Parser Bool -parseGroupFailFast = - OA.switch - ( mconcat +parseGroupFailFast = mkSwitch opts + where + opts = + mconcat [ OA.long "group-fail-fast", + OA.value False, mkHelp helpTxt ] - ) - where helpTxt = mconcat - [ "If true, the first batch group that fails to completely build stops ", - "clc-stackage." + [ "If on, the first batch group that fails to completely build stops ", + "clc-stackage. Defaults to 'off'." ] -parseNoCabalUpdate :: Parser Bool -parseNoCabalUpdate = - OA.switch - ( mconcat - [ OA.long "no-cabal-update", +parseCabalUpdate :: Parser Bool +parseCabalUpdate = mkSwitch opts + where + opts = + mconcat + [ OA.long "cabal-update", + OA.value True, mkHelpNoLine helpTxt ] - ) - where - helpTxt = "If true, skips the 'cabal update' step." + helpTxt = "Runs 'cabal update' before building. Defaults to 'on'." -parseNoCache :: Parser Bool -parseNoCache = - OA.switch - ( mconcat - [ OA.long "no-cache", +parseCache :: Parser Bool +parseCache = mkSwitch opts + where + opts = + mconcat + [ OA.long "cache", + OA.value True, mkHelp $ mconcat - [ "Disables the cache. Normally, the outcome of a run is saved ", - "to a json cache. This is useful for resuming a run that was ", - "interrupted (e.g. CTRL-C). The next run will fetch the ", - "packages to build from the cache." + [ "Saves the outcome of a run to a json cache, useful for resuming ", + "a run that was interrupted (e.g. CTRL-C). The next run will fetch ", + "the packages to build from the cache. Defaults to 'on'." ] ] - ) -parseNoCleanup :: Parser Bool -parseNoCleanup = - OA.switch - ( mconcat - [ OA.long "no-cleanup", - mkHelp "Will not remove the generated cabal files after exiting." +parseCleanup :: Parser Bool +parseCleanup = mkSwitch opts + where + opts = + mconcat + [ OA.long "cleanup", + OA.value True, + mkHelp "Removes generated files after finishing. Defaults to 'on'." ] - ) parsePackageFailFast :: Parser Bool -parsePackageFailFast = - OA.switch - ( mconcat +parsePackageFailFast = mkSwitch opts + where + opts = + mconcat [ OA.long "package-fail-fast", + OA.value False, mkHelpNoLine helpTxt ] - ) - where helpTxt = mconcat - [ "If true, the first package that fails _within_ a batch group ", + [ "If on, the first package that fails _within_ a batch group ", "will cause the entire group to fail. We then move to the next ", - "group, as normal. The default (off) behavior is equivalent to ", + "group, as normal. The default 'off' behavior is equivalent to ", "cabal's --keep-going)." ] +-- Notice that unlike other on/off switches, this is an actual flag +-- (--print-package-set) vs. an on/off option (--print-package-set (on | off)). +-- +-- We have this exception because this isn't really an on/off switch but +-- rather an alternative command which bypasses the build entirely. This +-- would make more sense using optparse's command syntax, except that would +-- require normal usage to also have some command (e.g. build), which doesn't +-- seem worth it for the normal, happy path. +-- +-- Ideally this would be a command and normal usage would be a "default command", +-- i.e. require no actual command, but optparse has no such notion. parsePrintPackageSet :: Parser Bool parsePrintPackageSet = OA.switch @@ -415,13 +427,18 @@ parsePrintPackageSet = ] parseRetryFailures :: Parser Bool -parseRetryFailures = - OA.switch - ( mconcat +parseRetryFailures = mkSwitch opts + where + opts = + mconcat [ OA.long "retry-failures", - mkHelpNoLine "Retries failures from the cache. Incompatible with --no-cache. " + OA.value False, + mkHelpNoLine $ + mconcat + [ "Retries failures from the cache. Incompatible with '--cache off'. ", + "Defaults to 'off'." + ] ] - ) parseSnapshotPath :: Parser (Maybe OsPath) parseSnapshotPath = @@ -561,3 +578,21 @@ cwdPathsCompleter = OAC.mkCompleter $ \word -> do tryIO :: IO a -> IO (Either IOException a) tryIO = try + +-- Makes a switch that takes '(on | off)'. For consistency, this should be +-- preferred for any on/off switch, rather than a normal flag +-- (e.g. --foo (on | off) vs. --foo). +mkSwitch :: Mod OptionFields Bool -> Parser Bool +mkSwitch opts = OA.option readSwitch opts' + where + opts' = + OA.metavar "(on | off)" + <> OA.completeWith ["on", "off"] + <> opts + +readSwitch :: ReadM Bool +readSwitch = + OA.str >>= \case + "off" -> pure False + "on" -> pure True + other -> fail $ "Expected (on | off), received: " ++ other diff --git a/src/CLC/Stackage/Runner/Env.hs b/src/CLC/Stackage/Runner/Env.hs index d5b3104..ed6f2c9 100644 --- a/src/CLC/Stackage/Runner/Env.hs +++ b/src/CLC/Stackage/Runner/Env.hs @@ -46,7 +46,7 @@ import CLC.Stackage.Utils.Logging qualified as Logging import CLC.Stackage.Utils.Package (Package (MkPackage, name, version)) import CLC.Stackage.Utils.Paths qualified as Paths import Control.Exception (throwIO) -import Control.Monad (join, unless, when) +import Control.Monad (join, when) import Data.Bool (Bool (False, True), not) import Data.Foldable (Foldable (foldl')) import Data.IORef (newIORef, readIORef) @@ -67,20 +67,20 @@ import Prelude (IO, Monad ((>>=)), mconcat, pure, show, ($), (.), (<$>), (<>)) data RunnerEnv = MkRunnerEnv { -- | Environment used in building. buildEnv :: BuildEnv, + -- | Enables the 'cabal update' step. + cabalUpdate :: Bool, -- | Status from previous run. cache :: Maybe Results, + -- | Enables the cache, which saves the outcome of a run in a json file. + -- The cache is used for resuming a run that was interrupted. + cacheEnabled :: Bool, + -- | If disabled, we do not revert the cabal file at the end (i.e. we + -- leave the last attempted build). + cleanup :: Bool, -- | The complete package set from stackage. This is used to write the -- cabal.project.local's constraint section, to ensure we always use the -- same transitive dependencies. completePackageSet :: [Package], - -- | Disables the 'cabal update' step. - noCabalUpdate :: Bool, - -- | Disables the cache, which otherwise saves the outcome of a run in a - -- json file. The cache is used for resuming a run that was interrupted. - noCache :: Bool, - -- | If we do not revert the cabal file at the end (i.e. we leave the - -- last attempted build). - noCleanup :: Bool, -- | Whether to retry packages that failed. retryFailures :: Bool, -- | Start time. @@ -136,9 +136,9 @@ setup hLoggerRaw modifyPackages = do failuresRef <- newIORef Set.empty cache <- - if cliArgs.noCache - then pure Nothing - else Report.readCache hLogger + if cliArgs.cache + then Report.readCache hLogger + else pure Nothing -- (entire set, packages to build) (completePackageSet, pkgsList) <- case cache of @@ -199,9 +199,9 @@ setup hLoggerRaw modifyPackages = do { buildEnv, cache, completePackageSet, - noCabalUpdate = cliArgs.noCabalUpdate, - noCache = cliArgs.noCache, - noCleanup = cliArgs.noCleanup, + cabalUpdate = cliArgs.cabalUpdate, + cacheEnabled = cliArgs.cache, + cleanup = cliArgs.cleanup, retryFailures = cliArgs.retryFailures, startTime } @@ -216,7 +216,7 @@ setup hLoggerRaw modifyPackages = do teardown :: RunnerEnv -> IO () teardown env = do endTime <- env.buildEnv.hLogger.getLocalTime - unless env.noCleanup $ do + when env.cleanup $ do Dir.removeFile Paths.generatedCabalPath Dir.removeFile Paths.generatedCabalProjectLocalPath @@ -227,7 +227,7 @@ teardown env = do (Logging.formatLocalTime env.startTime) (Logging.formatLocalTime endTime) - unless env.noCache (updateCache env results) + when env.cacheEnabled (updateCache env results) Report.saveReport report diff --git a/test/functional/Main.hs b/test/functional/Main.hs index 95d685e..697fc5a 100644 --- a/test/functional/Main.hs +++ b/test/functional/Main.hs @@ -161,10 +161,13 @@ runGolden getNoCleanup params = -- While NOTE: [Skipping cleanup] will prevent the test cleanup from running, -- the clc-stackage also performs a cleanup. Thus if no cleanup is desired - -- (NO_CLEANUP is set), we also need to pass the --no-cleanup arg to the + -- (NO_CLEANUP is set), we also need to pass the '--cleanup off' arg to the -- exe. noCleanup <- getNoCleanup - let noCleanupArgs = ["--no-cleanup" | noCleanup] + let noCleanupArgs = + if noCleanup + then [] + else ["--cleanup", "off"] finalArgs = args' ++ noCleanupArgs logs <- withArgs finalArgs params.runner @@ -245,7 +248,8 @@ runGolden getNoCleanup params = args' = "--color-logs" : "off" - : "--no-cabal-update" + : "--cabal-update" + : "off" : params.args baseTestPath = diff --git a/test/unit/Unit/Prelude.hs b/test/unit/Unit/Prelude.hs index 2d7f94d..c9f9ae0 100644 --- a/test/unit/Unit/Prelude.hs +++ b/test/unit/Unit/Prelude.hs @@ -23,11 +23,11 @@ import CLC.Stackage.Runner.Env ( RunnerEnv ( MkRunnerEnv, buildEnv, + cabalUpdate, cache, + cacheEnabled, + cleanup, completePackageSet, - noCabalUpdate, - noCache, - noCleanup, retryFailures, startTime ), @@ -51,9 +51,9 @@ mkRunnerEnv = do { buildEnv, cache = Nothing, completePackageSet = NE.toList buildEnv.packagesToBuild, - noCabalUpdate = True, - noCache = False, - noCleanup = False, + cabalUpdate = False, + cacheEnabled = True, + cleanup = True, retryFailures = False, startTime = mkLocalTime } From aa763ff6c325603df3335ec128386cab963164b5 Mon Sep 17 00:00:00 2001 From: Tommy Bidne Date: Mon, 6 Apr 2026 12:22:14 +1200 Subject: [PATCH 4/5] Add real build to CI Adds the build-batch job to CI which actually builds the entire snapshot. It does this by dividing the snapshot into batch groups per --batch, then having a different job build each group via github action's matrix strategy and new clc-stackage --batch-index param. This allows us to build the entire snapshot such that overall CI time is reasonable i.e. ~ 1 hour. --- .github/scripts/batch_index.sh | 62 +++++++++++++++++++++++++ .github/scripts/dry_run.sh | 22 +++++---- .github/workflows/ci.yaml | 81 +++++++++++++++++++++++++++++++-- README.md | 2 +- src/CLC/Stackage/Builder/Env.hs | 4 ++ src/CLC/Stackage/Runner.hs | 38 +++++++++++++++- src/CLC/Stackage/Runner/Args.hs | 26 ++++++++++- src/CLC/Stackage/Runner/Env.hs | 7 ++- test/unit/Unit/Prelude.hs | 2 + 9 files changed, 224 insertions(+), 20 deletions(-) create mode 100755 .github/scripts/batch_index.sh diff --git a/.github/scripts/batch_index.sh b/.github/scripts/batch_index.sh new file mode 100755 index 0000000..69ac9fc --- /dev/null +++ b/.github/scripts/batch_index.sh @@ -0,0 +1,62 @@ +set -e + +# Similar to dry_run.sh, except actually builds exactly one batch group. +# This way CI can spread the full build across multiple jobs, keeping the +# total time reasonable. +batch_index=$1 + +# -f not -x since downloaded exe may not have executable permissions. +if [[ -f ./bin/clc-stackage ]]; then + echo "*** ./bin/clc-stackage exists, not re-installing ***" + + # May need to add permissions, if this exe was downloaded + chmod a+x ./bin/clc-stackage +else + echo "*** Updating cabal ***" + cabal update + + echo "*** Installing clc-stackage ***" + cabal install exe:clc-stackage --installdir=./bin --overwrite-policy=always +fi + +if [[ -d output ]]; then + rm -r output +fi + +echo "*** Building with --batch-index $batch_index ***" + +set +e + +./bin/clc-stackage \ + --batch 200 \ + --batch-index $batch_index \ + --cabal-options="--semaphore" \ + --cleanup off + +ec=$? + +if [[ $ec != 0 ]]; then + echo "*** clc-stackage failed ***" +else + echo "*** clc-stackage succeeded ***" +fi + +# Print out the logs + the packages we built, in case it is useful e.g. +# what did CI actually do. +if [[ -f generated/generated.cabal ]]; then + echo "*** Printing generated cabal file ***" + cat generated/generated.cabal +else + echo "*** No generated/generated.cabal ***" +fi + +if [[ -f generated/cabal.project.local ]]; then + echo "*** Printing generated cabal.project.local file ***" + cat generated/cabal.project.local +else + echo "*** No generated/cabal.project.local ***" +fi + +.github/scripts/print_logs.sh + +exit $ec diff --git a/.github/scripts/dry_run.sh b/.github/scripts/dry_run.sh index a2243b2..67f05c7 100755 --- a/.github/scripts/dry_run.sh +++ b/.github/scripts/dry_run.sh @@ -1,15 +1,19 @@ set -e -echo "*** Updating cabal ***" - -cabal update +if [[ -f ./bin/clc-stackage ]]; then + echo "*** ./bin/clc-stackage exists, not re-installing ***" + chmod a+x ./bin/clc-stackage +else + echo "*** Updating cabal ***" + cabal update -echo "*** Installing clc-stackage ***" + # --overwrite-policy=always and deleting output/ are unnecessary for CI since + # this script will only be run one time, but it's helpful when we are + # testing the script locally. -# --overwrite-policy=always and deleting output/ are unnecessary for CI since -# this script will only be run one time, but it's helpful when we are -# testing the script locally. -cabal install exe:clc-stackage --installdir=./bin --overwrite-policy=always + echo "*** Installing clc-stackage ***" + cabal install exe:clc-stackage --installdir=./bin --overwrite-policy=always +fi if [[ -d output ]]; then rm -r output @@ -18,7 +22,7 @@ fi echo "*** Building all with --dry-run ***" set +e -./bin/clc-stackage --batch 100 --cabal-options="--dry-run" +./bin/clc-stackage --batch 200 --cabal-options="--dry-run" ec=$? diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1de8319..ecc0b4a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -26,7 +26,7 @@ jobs: - "windows-latest" runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - uses: haskell-actions/setup@v2 with: # Should be the current stackage nightly, though this will likely go @@ -57,7 +57,7 @@ jobs: if: ${{ failure() && steps.functional.conclusion == 'failure' }} shell: bash run: .github/scripts/print_logs.sh - nix: + dry-run: strategy: fail-fast: false matrix: @@ -65,13 +65,86 @@ jobs: - "ubuntu-latest" runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Setup nix - uses: cachix/install-nix-action@v30 + uses: cachix/install-nix-action@v31 with: github_access_token: ${{ secrets.GITHUB_TOKEN }} nix_path: nixpkgs=channel:nixos-unstable - name: Dry run run: nix develop .#ci -Lv -c bash -c '.github/scripts/dry_run.sh' + + # Upload installed binary so that build-batch does not need to re-install + # it. + - name: Upload clc-stackage binary + uses: actions/upload-artifact@v7 + with: + name: clc-stackage-binary + path: ./bin/clc-stackage + retention-days: 1 + + # Uses jq's 'range(m; n)' operator to create list of indexes from [m, n) + # for the build-batch job. Slightly nicer than manually listing all of them. + build-batch-indexes: + runs-on: "ubuntu-latest" + outputs: + indexes: ${{ steps.set-batch-indexes.outputs.indexes }} + steps: + - id: set-batch-indexes + run: echo "indexes=$(jq -cn '[range(1; 19)]')" >> $GITHUB_OUTPUT + + # Ideally CI would run a job that actually builds all packages, but this + # can take a very long time, potentially longer than github's free CI limits + # (last time checked: 5.5 hrs). + # + # What we can do instead, is perform the usual batch process of dividing the + # package set into groups, then have a different job build each group. + # This does /not/ run up against github's free CI limits. + # + # To do this, we have the script batch_index.sh divide the package set into + # groups, per --batch. Then, using github's matrix strategy, have each + # job build only a specific group by passing its index as --batch-index. + # + # In other words, each job runs + # + # clc-stackage --batch N --batch-index k + # + # where k is matrix.index, hence each building a different group. + # The only other consideration we have, then, is to make sure we have enough + # indices to cover the whole package set. + # + # Currently, we choose --batch to be 200, and the total package set is + # around 3400, which is filtered to about 3100 packages to build. We thus + # need at least ceiling(3100 / 200) = 16 indexes to cover this. + # + # There is no harm in going overboard e.g. if we have an index that is out of + # range, that job will simply end with a warning message. We should + # therefore err on the side of adding too many indices, rather than too few. + build-batch: + needs: [build-batch-indexes, dry-run] + strategy: + fail-fast: false + matrix: + index: ${{ fromJSON(needs.build-batch-indexes.outputs.indexes) }} + name: Batch group ${{ matrix.index }} + runs-on: "ubuntu-latest" + steps: + - uses: actions/checkout@v6 + + - name: Setup nix + uses: cachix/install-nix-action@v31 + with: + github_access_token: ${{ secrets.GITHUB_TOKEN }} + nix_path: nixpkgs=channel:nixos-unstable + + # Download clc-stackage binary from dry-run job. + - name: Download binary + uses: actions/download-artifact@v7 + with: + name: clc-stackage-binary + path: ./bin + + - name: Build + run: nix develop .#ci -Lv -c bash -c '.github/scripts/batch_index.sh ${{ matrix.index }}' diff --git a/README.md b/README.md index c4d8824..f7d60be 100644 --- a/README.md +++ b/README.md @@ -119,7 +119,7 @@ In general, user mitigations for solver / build problems include: compiler = pkgs.haskell.packages.ghc; ``` - can be a useful guide as to which GHC was last tested, as CI uses this ghc to build everything with `--dry-run`, which should report solver errors (e.g. bounds) at the very least. + can be a useful guide as to which GHC was last tested, as CI uses this ghc to build everything. - If you encounter an error that you think indicates a problem with the configuration here (e.g. new package needs to be excluded, new constraint added), please open an issue. While that is being resolved, the mitigations from the [previous section](#troubleshooting) may be useful. diff --git a/src/CLC/Stackage/Builder/Env.hs b/src/CLC/Stackage/Builder/Env.hs index a6c565e..5a27e6f 100644 --- a/src/CLC/Stackage/Builder/Env.hs +++ b/src/CLC/Stackage/Builder/Env.hs @@ -34,6 +34,10 @@ data BuildEnv = MkBuildEnv { -- | If we have @Just n@, 'packagesToBuild' will be split into groups of at most -- size @n@. If @Nothing@, the entire set will be built in one go. batch :: Maybe Int, + -- | 1-based index for building the Nth package group only, according to + -- --batch. Intended for CI use, where building all groups takes too much + -- time. + batchIndex :: Maybe Int, -- | Build arguments for cabal. buildArgs :: [String], -- | Optional path to cabal executable. diff --git a/src/CLC/Stackage/Runner.hs b/src/CLC/Stackage/Runner.hs index 804e22d..a4aa246 100644 --- a/src/CLC/Stackage/Runner.hs +++ b/src/CLC/Stackage/Runner.hs @@ -8,7 +8,7 @@ where import CLC.Stackage.Builder qualified as Builder import CLC.Stackage.Builder.Env - ( BuildEnv (hLogger, progress), + ( BuildEnv (batchIndex, hLogger, progress), Progress (failuresRef), ) import CLC.Stackage.Builder.Writer qualified as Writer @@ -20,6 +20,10 @@ import Control.Exception (bracket, throwIO) import Control.Monad (when) import Data.Foldable (for_) import Data.IORef (readIORef) +import Data.List.NonEmpty (NonEmpty) +import Data.List.NonEmpty qualified as NE +import Data.Text (Text) +import Data.Text qualified as T import System.Exit (ExitCode (ExitFailure)) import System.IO qualified as IO @@ -52,7 +56,24 @@ runModifyPackages hLogger modifyPackages = withHiddenInput $ do Logging.putTimeInfoStr buildEnv.hLogger "Starting build(s)" - for_ pkgGroupsIdx $ \(pkgGroup, idx) -> Builder.buildProject buildEnv idx pkgGroup + case buildEnv.batchIndex of + -- No batch index: normal, build all groups sequentially. + Nothing -> for_ pkgGroupsIdx $ \(pkgGroup, idx) -> + Builder.buildProject buildEnv idx pkgGroup + Just batchIndex -> + -- Some batch index: if it is in range, build that group. + case index batchIndex pkgGroupsIdx of + Just (pkgGroup, idx) -> Builder.buildProject buildEnv idx pkgGroup + Nothing -> do + let msg = + mconcat + [ "Nothing to build. Index '", + showt batchIndex, + "' is out of range for ", + showt $ NE.length pkgGroupsIdx, + " group(s)." + ] + Logging.putTimeWarnStr buildEnv.hLogger msg numErrors <- length <$> readIORef buildEnv.progress.failuresRef when (numErrors > 0) $ throwIO $ ExitFailure 1 @@ -75,3 +96,16 @@ withHiddenInput m = bracket hideInput unhideInput (const m) unhideInput (buffMode, echoMode) = do IO.hSetBuffering IO.stdin buffMode IO.hSetEcho IO.stdin echoMode + +index :: Int -> NonEmpty a -> Maybe a +index idx = go idx' . NE.toList + where + go _ [] = Nothing + go 0 (x : _) = Just x + go !n (_ : xs) = go (n - 1) xs + + -- Subtract one since the index is one-based, not zero. + idx' = idx - 1 + +showt :: (Show a) => a -> Text +showt = T.pack . show diff --git a/src/CLC/Stackage/Runner/Args.hs b/src/CLC/Stackage/Runner/Args.hs index c1cb23e..38ae88b 100644 --- a/src/CLC/Stackage/Runner/Args.hs +++ b/src/CLC/Stackage/Runner/Args.hs @@ -58,6 +58,10 @@ data Args = MkArgs { -- | If given, batches packages together so we build more than one. -- Defaults to batching everything together in the same group. batch :: Maybe Int, + -- | 1-based index for building the Nth package group only, according to + -- --batch. Intended for CI use, where building all groups takes too much + -- time. + batchIndex :: Maybe Int, -- | Global options to pass to cabal e.g. --store-dir. cabalGlobalOpts :: [String], -- | Options to pass to cabal e.g. --semaphore. @@ -191,12 +195,13 @@ parseCliArgs = ~(cabalGlobalOpts, cabalOpts, cabalPath, cabalUpdate) <- parseCabalGroup ~(cache, retryFailures) <- parseCacheGroup ~(groupFailFast, packageFailFast) <- parseFailuresGroup - ~(batch, printPackageSet, snapshotPath) <- parseMiscGroup + ~(batch, batchIndex, printPackageSet, snapshotPath) <- parseMiscGroup ~(cleanup, colorLogs, writeLogs) <- parseOutputGroup pure $ MkArgs { batch, + batchIndex, cabalGlobalOpts, cabalOpts, cabalPath, @@ -236,8 +241,9 @@ parseCliArgs = parseMiscGroup = OA.parserOptionGroup "Misc options:" $ - (,,) + (,,,) <$> parseBatch + <*> parseBatchIndex <*> parsePrintPackageSet <*> parseSnapshotPath @@ -267,6 +273,22 @@ parseBatch = ] ) +-- Determines which --batch group to build. Normally we want to build all +-- groups, so this arg is intended only for CI, where a single CI job cannot +-- build everything, or it will time out. Hence this is marked 'internal' +-- to hide it from the --help page, as its presence would only confuse. +parseBatchIndex :: Parser (Maybe Int) +parseBatchIndex = + OA.optional $ + OA.option + OA.auto + ( mconcat + [ OA.long "batch-index", + OA.internal, + OA.metavar "NAT" + ] + ) + parseCabalGlobalOpts :: Parser [String] parseCabalGlobalOpts = OA.option diff --git a/src/CLC/Stackage/Runner/Env.hs b/src/CLC/Stackage/Runner/Env.hs index ed6f2c9..6fa7a53 100644 --- a/src/CLC/Stackage/Runner/Env.hs +++ b/src/CLC/Stackage/Runner/Env.hs @@ -181,6 +181,7 @@ setup hLoggerRaw modifyPackages = do buildEnv = MkBuildEnv { batch = cliArgs.batch, + batchIndex = cliArgs.batchIndex, buildArgs, cabalPath, groupFailFast = cliArgs.groupFailFast, @@ -217,8 +218,10 @@ teardown :: RunnerEnv -> IO () teardown env = do endTime <- env.buildEnv.hLogger.getLocalTime when env.cleanup $ do - Dir.removeFile Paths.generatedCabalPath - Dir.removeFile Paths.generatedCabalProjectLocalPath + -- removePathForcibly as these might not exist e.g. if the we did not + -- build anything. + Dir.removePathForcibly Paths.generatedCabalPath + Dir.removePathForcibly Paths.generatedCabalProjectLocalPath results <- getResults env.buildEnv let report = diff --git a/test/unit/Unit/Prelude.hs b/test/unit/Unit/Prelude.hs index c9f9ae0..edfed4e 100644 --- a/test/unit/Unit/Prelude.hs +++ b/test/unit/Unit/Prelude.hs @@ -9,6 +9,7 @@ import CLC.Stackage.Builder.Env ( BuildEnv ( MkBuildEnv, batch, + batchIndex, buildArgs, cabalPath, groupFailFast, @@ -66,6 +67,7 @@ mkBuildEnv = do pure $ MkBuildEnv { batch = Nothing, + batchIndex = Nothing, buildArgs = [], cabalPath = "cabal", groupFailFast = False, From 2276a6fade6106abcb2f6d1fe12a717081f31f8e Mon Sep 17 00:00:00 2001 From: Tommy Bidne Date: Tue, 7 Apr 2026 14:51:16 +1200 Subject: [PATCH 5/5] Split excluded_pkgs into separate groups Normally, each package in a given package set is used in two ways: - It is written to generated.cabal's build-depends, hence built. - Its exact version is written to cabal.project.local as a constraint, for reproducibility of transitive dependencies. We used excluded_pkgs.jsonc to opt out of one or both of these for a given package e.g. - Totally exclude cuda from the build. - Exclude boot libs like text from having their exact versions written, as these libs will be used transitively anyway, and exact pins complicate building with multiple GHCs. We since stumbled onto a third scenario: packages that we do not want in build-depends yet benefit from being pinned. For example, we do not want happy in build-depends (since it is an exe), yet we would like to use stackage's pins, so that cabal does not choose a version that might not work. We therefore modify excluded_pkgs.jsonc to package_index.jsonc, which separates these categories into different json keys: - excluded: Same as before, these packages are filtered out from the given package set. - excluded_pinned: Packages that are filtered from the package set yet we want their pins written anyway e.g. happy. - unpinned: Packages to be built but should not be pinned e.g. boot libs. --- README.md | 8 +- dev.md | 8 +- excluded_pkgs.jsonc | 433 ---------------------------- package_index.jsonc | 447 +++++++++++++++++++++++++++++ src/CLC/Stackage/Builder/Writer.hs | 28 +- src/CLC/Stackage/Parser.hs | 173 +++++++++-- src/CLC/Stackage/Runner/Args.hs | 2 +- src/CLC/Stackage/Runner/Env.hs | 26 +- test/unit/Unit/Prelude.hs | 15 +- 9 files changed, 657 insertions(+), 483 deletions(-) delete mode 100644 excluded_pkgs.jsonc create mode 100644 package_index.jsonc diff --git a/README.md b/README.md index f7d60be..b48361b 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ Because we build with `nightly` and are at the mercy of cabal's constraint solve - `p` requires a new system dependency (e.g. a C library). - `p` is an executable. - - `p` depends on a package in [./excluded_pkgs.jsonc](excluded_pkgs.jsonc). + - `p` depends on an excluded package in [./package_index.jsonc](package_index.jsonc). - A cabal flag is set in a way that breaks the build. For example, our snapshot requires that the `bson` library does *not* have its `_old-network` flag set, as this will cause a build error with our version of `network`. This flag is automatic, so we have to force it in `generated/cabal.project` with `constraints: bson -_old-network`. @@ -95,7 +95,7 @@ We attempt to mitigate such issues by: Nevertheless, it is still possible for issues to slip through. When a package `p` fails to build for some reason, we should first: -- Verify that `p` is not in `excluded_pkgs.jsonc`. If it is, nightly probably pulled in some new reverse-dependency `q` that should be added to `excluded_pkgs.jsonc`. +- Verify that `p` is not in `package_index.excluded`. If it is, nightly probably pulled in some new reverse-dependency `q` that should be added to `package_index.excluded`. - Verify that `p` does not have cabal flags that can affect dependencies / API. @@ -103,9 +103,9 @@ Nevertheless, it is still possible for issues to slip through. When a package `p In general, user mitigations for solver / build problems include: -- Adding `p` to `excluded_pkgs.jsonc`. Note that `p` will still be built if it is a (transitive) dependency of some other package in the snapshot, but will not have its exact bounds written to `cabal.project.local`. +- Adding `p` to `package_index.excluded`. Note that `p` will still be built if it is a (transitive) dependency of some other package in the snapshot, but will not have its exact bounds written to `cabal.project.local`. -- Manually downloading a snapshot (e.g. `https://www.stackage.org/nightly/cabal.config`), changing / removing the offending package(s), and supplying the file with the `--snapshot-path` param. Like `excluded_pkgs.jsonc`, take care that the problematic package is not a (transitive) dependency of something in the snapshot. +- Manually downloading a snapshot (e.g. `https://www.stackage.org/nightly/cabal.config`), changing / removing the offending package(s), and supplying the file with the `--snapshot-path` param. Like `package_index.jsonc`, take care that the problematic package is not a (transitive) dependency of something in the snapshot. - Adding constraints to `generated/cabal.project` e.g. flags or version constraints like `constraints: filepath > 1.5`. diff --git a/dev.md b/dev.md index 48d9c63..3a5cdab 100644 --- a/dev.md +++ b/dev.md @@ -20,7 +20,7 @@ The `clc-stackage` library is namespaced by functionality: ### parser -`CLC.Stackage.Parser` contains the parsing functionality. In particular, `parser` is responsible for querying stackage's REST endpoint and retrieving the package set. That package set is then filtered according to [excluded_pkgs.json](excluded_pkgs.json). The primary function is: +`CLC.Stackage.Parser` contains the parsing functionality. In particular, `parser` is responsible for querying stackage's REST endpoint and retrieving the package set. That package set is then filtered according to [package_index.jsonc](package_index.jsonc). The primary function is: ```haskell -- CLC.Stackage.Parser @@ -77,7 +77,7 @@ The executable that actually runs. This is a very thin wrapper over `runner`, wh `clc-stackage` is based on `nightly` -- which changes automatically -- meaning we do not necessarily have to do anything when a new (minor) snapshot is released. On the other hand, *major* snapshot updates will almost certainly bring in new packages that need to be excluded, so there are some general "update steps" we will want to take: -1. Modify [excluded_pkgs.json](excluded_pkgs.json) as needed. That is, updating the snapshot major version will probably bring in some new packages that we do not want. The update process is essentially trial-and-error i.e. run `clc-stackage` as normal, and later add any failing packages that should be excluded. +1. Modify [package_index.jsonc](package_index.jsonc) as needed. That is, updating the snapshot major version will probably bring in some new packages that we do not want. The update process is essentially trial-and-error i.e. run `clc-stackage` as normal, and later add any failing packages to `package_index.excluded` that should be excluded. 2. Update `ghc-version` in [.github/workflows/ci.yaml](.github/workflows/ci.yaml). @@ -112,3 +112,7 @@ $ NO_CLEANUP=1 cabal test functional ``` Note that this only saves files from the _last_ test, so if you want to examine test output for a particular test, you need to run only that test. + +> [!TIP] +> +> CI has a job `build-batch` which actually builds the entire package set, hence it can be used in place of manual building / testing. Note it takes about an hour to run. diff --git a/excluded_pkgs.jsonc b/excluded_pkgs.jsonc deleted file mode 100644 index 56e60fb..0000000 --- a/excluded_pkgs.jsonc +++ /dev/null @@ -1,433 +0,0 @@ -{ - "all": [ - "agda2lagda", - "al", - "alex", - "align-audio", - "Allure", - "alsa-core", - "alsa-mixer", - "alsa-pcm", - "alsa-seq", - "ALUT", - "amqp-utils", - "arbtt", - "array", // see NOTE: [Boot packages] - // NOTE: [Boot packages] - // - // Boot packages are excluded from directly building here -- and having - // their constraints written -- because - // their versions are not stable within the same major ghc version, - // and a version mismatch will cause a build failure whenever ghc - // is in the build plan (which is true for stackage). - // - // Packages taken from: - // https://gitlab.haskell.org/ghc/ghc/-/wikis/commentary/libraries/version-history - "base", - "beam-postgres", - "bench", - "bhoogle", - "binary", // see NOTE: [Boot packages] - "bindings-libzip", - "blas-carray", - "blas-comfort-array", - "blas-ffi", - "boomwhacker", - "btrfs", - "buffer-pipe", - "bugsnag", - "bugsnag-wai", - "bugsnag-yesod", - "bytestring", // see NOTE: [Boot packages] - "c2hs", - "Cabal", // see NOTE: [Boot packages] - "cabal-clean", - "cabal-flatpak", - "cabal-install", - "cabal-install-solver", - "cabal-rpm", - "cabal-sort", - "Cabal-syntax", // see NOTE: [Boot packages] - "cabal2nix", - "calendar-recycling", - "Clipboard", - "coinor-clp", - "comfort-blas", - "comfort-fftw", - "comfort-glpk", - "containers", // see NOTE: [Boot packages] - "core-telemetry", - "countdown-numbers-game", - "cql-io", - "crackNum", - "cryptonite-openssl", - "cuda", - "cutter", - "dbcleaner", - "dbus-menu", // gtk-3 - "deepseq", // see NOTE: [Boot packages] - "diagrams-svg", - "directory", // see NOTE: [Boot packages] - "discount", - "dl-fedora", - "doctest-extract", - "drifter-postgresql", - "Ebnf2ps", - "elynx", - "emd", - "equal-files", - "essence-of-live-coding-pulse", - "exceptions", // see NOTE: [Boot packages] - "experimenter", - "fbrnch", - "fedora-haskell-tools", - "fedora-repoquery", - "fft", - "fftw-ffi", - "file-io", // see NOTE: [Boot packages] - "file-modules", - "filepath", // see NOTE: [Boot packages] - "fix-whitespace", - "flac", - "flac-picture", - "follow-file", - "freckle-app", - "freenect", - "fsnotify", - "fsnotify-conduit", - "gauge", - "gd", - "ghc", // see NOTE: [Boot packages] - "ghc-bignum", // see NOTE: [Boot packages] - "ghc-boot", // see NOTE: [Boot packages] - "ghc-boot-th", // see NOTE: [Boot packages] - "ghc-compact", // see NOTE: [Boot packages] - "ghc-core", - "ghc-experimental", // see NOTE: [Boot packages] - "ghc-heap", // see NOTE: [Boot packages] - "ghc-internal", // see NOTE: [Boot packages] - "ghc-platform", // see NOTE: [Boot packages] - "ghc-prim", - "ghc-syntax-highlighter", - "ghc-toolchain", // see NOTE: [Boot packages] - "ghci", // see NOTE: [Boot packages] - "ghostscript-parallel", - "gi-atk", - "gi-cairo", - "gi-cairo-connector", - "gi-cairo-render", - "gi-dbusmenu", - "gi-dbusmenugtk3", - "gi-freetype2", - "gi-gdk", - "gi-gdk3", - "gi-gdk4", - "gi-gdkpixbuf", - "gi-gdkx11", - "gi-gdkx113", - "gi-gdkx114", - "gi-gio", - "gi-glib", - "gi-gmodule", - "gi-gobject", - "gi-graphene", - "gi-gsk", - "gi-gtk", - "gi-gtk3", - "gi-gtk4", - "gi-gtk-hs", - "gi-gtk-layer-shell", - "gi-gtksource", - "gi-gtksource5", - "gi-harfbuzz", - "gi-javascriptcore", - "gi-javascriptcore4", - "gi-javascriptcore6", - "gi-pango", - "gi-soup", - "gi-soup2", - "gi-soup3", - "gi-vte", - "gi-webkit2", - "gi-xlib", - "git-annex", - "git-mediate", - "gl", - "gloss-examples", - "glpk-headers", - "goldplate", - "google-oauth2-jwt", - "gopher-proxy", - "group-by-date", - "gtk", - "gtk-scaling-image", - "gtk-sni-tray", - "gtk-strut", - "gtk2hs-buildtools", - "gtk3", - "H", - "hackage-cli", - "haddock-api", // see NOTE: [Boot packages] - "haddock-library", // see NOTE: [Boot packages] - "hamtsolo", - "happstack-server-tls", - "happy", - "haskeline", // see NOTE: [Boot packages] - "haskell-gi", - "haskell-gi-base", - "haskell-gi-overloading", - "haskoin-core", - "haskoin-node", - "haskoin-store", - "haskoin-store-data", - "hasql", - "hasql-dynamic-statements", - "hasql-implicits", - "hasql-interpolate", - "hasql-listen-notify", - "hasql-migration", - "hasql-notifications", - "hasql-optparse-applicative", - "hasql-pool", - "hasql-queue", - "hasql-th", - "hasql-transaction", - "haxr", - "headroom", // segfault - "hinotify", - "hkgr", - "hledger-interest", - "hledger-ui", - "hlibgit2", - "hmatrix-gsl", - "hmatrix-gsl-stats", - "hmatrix-special", - "hmm-lapack", - "hmpfr", - "hpc", // see NOTE: [Boot packages] - "hopenssl", - "hp2pretty", - "hpqtypes", - "hpqtypes-extras", - "hruby", - "hs-captcha", - "hs-GeoIP", - "hsc2hs", - "hsdns", - "hsignal", - "hsndfile", - "hsndfile-vector", - "HsOpenSSL", - "HsOpenSSL-x509-system", - "hsshellscript", - "hstatistics", - "htaglib", - "http-client-openssl", - "http-io-streams", - "http-streams", - "hw-json-simd", - "hw-kafka-client", - "hwk", - "ihaskell", - "ihaskell-hvega", - "ihs", - "Imlib", - "inline-r", - "integer-gmp", // see NOTE: [Boot packages] - "integer-simple", - "ip6addr", - "ipython-kernel", - "jack", - "jailbreak-cabal", - "java-adt", - "JuicyPixels-scale-dct", - "koji", - "koji-tool", - "krank", - "LambdaHack", - "lame", - "lapack", - "lapack-carray", - "lapack-comfort-array", - "lapack-ffi", - "lapack-ffi-tools", - "lapack-hmatrix", - "lens-regex-pcre", - "lentil", - "leveldb-haskell", - "liboath-hs", - "linear-circuit", - "linux-file-extents", - "linux-namespaces", - "lmdb", - "lsfrom", - "lzma-clib", - "magic", - "magico", - "mbox-utility", - "mega-sdist", - "microformats2-parser", // segfault - "midi-alsa", - "midi-music-box", - "misfortune", - "mmark-cli", - "mmark-ext", // ghc-syntax-highlighter - "moffy-samples-gtk3", - "moffy-samples-gtk3-run", - "moffy-samples-gtk4", - "moffy-samples-gtk4-run", - "mpi-hs", - "mpi-hs-binary", - "mpi-hs-cereal", - "mstate", - "mtl", // see NOTE: [Boot packages] - "mysql", - "mysql-json-table", - "mysql-simple", - "nanovg", - "netcode-io", - "nfc", - "NineP", - "nix-paths", - "nvvm", - "odbc", - "ods2csv", - "ogma-cli", - "opaleye", - "openssl-streams", - "OrderedBits", - "os-string", // see NOTE: [Boot packages] - "pagure-cli", - "pandoc-cli", - "parsec", // see NOTE: [Boot packages] - "parser-combinators-tests", - "pcre-heavy", - "pcre-light", - "peregrin", - "perf", - "persistent-mysql", - "persistent-postgresql", - "pg-transact", - "pkgtreediff", - "place-cursor-at", - "pontarius-xmpp", - "pontarius-xmpp-extras", - "postgresql-libpq-notify", - "postgresql-migration", - "postgresql-schema", - "postgresql-simple", - "postgresql-simple-url", - "pretty", // see NOTE: [Boot packages] - "primecount", - "process", // see NOTE: [Boot packages] - "profiterole", - "proto-lens-protobuf-types", - "psql-helpers", - "pthread", - "pulse-simple", - "rawfilepath", - "rdtsc", - "re2", - "reactive-balsa", - "reactive-jack", - "reanimate-svg", - "regex-pcre", - "rel8", - "resistor-cube", - "rex", - "rhbzquery", - "rocksdb-haskell", - "rocksdb-haskell-jprupp", - "rocksdb-query", - "rts", // see NOTE: [Boot packages] - "scrypt", - "sdl2", - "sdl2-gfx", - "sdl2-image", - "sdl2-mixer", - "sdl2-ttf", - "secp256k1-haskell", - "semaphore-compat", // see NOTE: [Boot packages] - "seqalign", - "servant-http-streams", - "servius", - "ses-html", - "shelltestrunner", - "snappy", - "sound-collage", - "soxlib", - "sphinx", - "split-record", - "sqlcli", - "sqlcli-odbc", - "sqlite-simple", - "stakhanov", // hasql - "stack-all", - "stack-clean-old", - "stack-templatizer", - "stm", // see NOTE: [Boot packages] - "stringprep", - "SVGFonts", - "swizzle", - "swizzle-lens", - "swizzle-modify", - "swizzle-set", - "sydtest-persistent-postgresql", - "synthesizer-alsa", - "taffybar", - "tasty-papi", - "template-haskell", // see NOTE: [Boot packages] - "termonad", - "terminfo", // see NOTE: [Boot packages] - "test-certs", - "text", // see NOTE: [Boot packages] - "text-icu", - "text-regex-replace", - "time", // see NOTE: [Boot packages] - "tls-debug", - "tmp-postgres", - "tmp-proc-postgres", - "transformers", // see NOTE: [Boot packages] - "ua-parser", - "uniq-deep", - "unix", // see NOTE: [Boot packages] - "users-postgresql-simple", - "validate-input", - "vector-fftw", - "visualize-type-inference", - "wai-session-postgresql", - "web3-tools", - "wild-bind-x11", - "Win32", // see NOTE: [Boot packages] - "Win32-notify", - "windns", - "X11", - "X11-xft", - "x11-xim", - "xhtml", // see NOTE: [Boot packages] - "xmonad", - "xmonad-contrib", - "xmonad-extras", - "yesod-bin", - "yi", - "yi-frontend-pango", - "yoga", - "youtube", - "zeromq4-haskell", - "zeromq4-patterns", - "zot", - "ztail" - ], - "linux": [ - "hfsevents", - "vty-windows" - ], - "osx": [ - "vty-windows" - ], - "windows": [ - "hfsevents", - "postgresql-libpq", - "vty-unix" - ] -} diff --git a/package_index.jsonc b/package_index.jsonc new file mode 100644 index 0000000..42cb5ac --- /dev/null +++ b/package_index.jsonc @@ -0,0 +1,447 @@ +{ + // All top-level keys are disjoint in the sense that a given package should + // belong to at most one. The notable exception is for packages that are + // in multiple excluded subkeys e.g. vty-windows in excluded.linux and + // excluded.osx. + + // "Excluded" packages are those that are excluded from the package set i.e. + // they will not be added to the generated/generated.cabal build-depends or + // have their versions pinned in generated/cabal.project.local. + "excluded": { + "all": [ + "agda2lagda", + "al", + "align-audio", + "Allure", + "alsa-core", + "alsa-mixer", + "alsa-pcm", + "alsa-seq", + "ALUT", + "amqp-utils", + "arbtt", + "beam-postgres", + "bench", + "bhoogle", + "bindings-libzip", + "blas-carray", + "blas-comfort-array", + "blas-ffi", + "boomwhacker", + "btrfs", + "buffer-pipe", + "bugsnag", + "bugsnag-wai", + "bugsnag-yesod", + "c2hs", + "cabal-clean", + "cabal-flatpak", + "cabal-install", + "cabal-install-solver", + "cabal-rpm", + "cabal-sort", + "cabal2nix", + "calendar-recycling", + "Clipboard", + "coinor-clp", + "comfort-blas", + "comfort-fftw", + "comfort-glpk", + "core-telemetry", + "countdown-numbers-game", + "cql-io", + "crackNum", + "cryptonite-openssl", + "cuda", + "cutter", + "dbcleaner", + "dbus-menu", // gtk-3 + "diagrams-svg", + "discount", + "dl-fedora", + "doctest-extract", + "drifter-postgresql", + "Ebnf2ps", + "elynx", + "emd", + "equal-files", + "essence-of-live-coding-pulse", + "experimenter", + "fbrnch", + "fedora-haskell-tools", + "fedora-repoquery", + "fft", + "fftw-ffi", + "file-modules", + "fix-whitespace", + "flac", + "flac-picture", + "follow-file", + "freckle-app", + "freenect", + "fsnotify", + "fsnotify-conduit", + "gauge", + "gd", + "ghc", + "ghc-bignum", + "ghc-boot", + "ghc-boot-th", + "ghc-compact", + "ghc-core", + "ghc-experimental", + "ghc-heap", + "ghc-internal", + "ghc-platform", + "ghc-prim", + "ghc-syntax-highlighter", + "ghc-toolchain", + "ghci", + "ghostscript-parallel", + "gi-atk", + "gi-cairo", + "gi-cairo-connector", + "gi-cairo-render", + "gi-dbusmenu", + "gi-dbusmenugtk3", + "gi-freetype2", + "gi-gdk", + "gi-gdk3", + "gi-gdk4", + "gi-gdkpixbuf", + "gi-gdkx11", + "gi-gdkx113", + "gi-gdkx114", + "gi-gio", + "gi-glib", + "gi-gmodule", + "gi-gobject", + "gi-graphene", + "gi-gsk", + "gi-gtk", + "gi-gtk3", + "gi-gtk4", + "gi-gtk-hs", + "gi-gtk-layer-shell", + "gi-gtksource", + "gi-gtksource5", + "gi-harfbuzz", + "gi-javascriptcore", + "gi-javascriptcore4", + "gi-javascriptcore6", + "gi-pango", + "gi-soup", + "gi-soup2", + "gi-soup3", + "gi-vte", + "gi-webkit2", + "gi-xlib", + "git-annex", + "git-mediate", + "gl", + "gloss-examples", + "glpk-headers", + "goldplate", + "google-oauth2-jwt", + "gopher-proxy", + "group-by-date", + "gtk", + "gtk-scaling-image", + "gtk-sni-tray", + "gtk-strut", + "gtk2hs-buildtools", + "gtk3", + "H", + "hackage-cli", + "hamtsolo", + "happstack-server-tls", + "haskell-gi", + "haskell-gi-base", + "haskell-gi-overloading", + "haskoin-core", + "haskoin-node", + "haskoin-store", + "haskoin-store-data", + "hasql", + "hasql-dynamic-statements", + "hasql-implicits", + "hasql-interpolate", + "hasql-listen-notify", + "hasql-migration", + "hasql-notifications", + "hasql-optparse-applicative", + "hasql-pool", + "hasql-queue", + "hasql-th", + "hasql-transaction", + "haxr", + "headroom", // segfault + "hinotify", + "hkgr", + "hledger-interest", + "hledger-ui", + "hlibgit2", + "hmatrix-gsl", + "hmatrix-gsl-stats", + "hmatrix-special", + "hmm-lapack", + "hmpfr", + "hopenssl", + "hp2pretty", + "hpqtypes", + "hpqtypes-extras", + "hruby", + "hs-captcha", + "hs-GeoIP", + "hsc2hs", + "hsdns", + "hsignal", + "hsndfile", + "hsndfile-vector", + "HsOpenSSL", + "HsOpenSSL-x509-system", + "hsshellscript", + "hstatistics", + "htaglib", + "http-client-openssl", + "http-io-streams", + "http-streams", + "hw-json-simd", + "hw-kafka-client", + "hwk", + "ihaskell", + "ihaskell-hvega", + "ihs", + "Imlib", + "inline-r", + "integer-simple", + "ip6addr", + "ipython-kernel", + "jack", + "jailbreak-cabal", + "java-adt", + "JuicyPixels-scale-dct", + "koji", + "koji-tool", + "krank", + "LambdaHack", + "lame", + "lapack", + "lapack-carray", + "lapack-comfort-array", + "lapack-ffi", + "lapack-ffi-tools", + "lapack-hmatrix", + "lens-regex-pcre", + "lentil", + "leveldb-haskell", + "liboath-hs", + "linear-circuit", + "linux-file-extents", + "linux-namespaces", + "lmdb", + "lsfrom", + "lzma-clib", + "magic", + "magico", + "mbox-utility", + "mega-sdist", + "microformats2-parser", // segfault + "midi-alsa", + "midi-music-box", + "misfortune", + "mmark-cli", + "mmark-ext", // ghc-syntax-highlighter + "moffy-samples-gtk3", + "moffy-samples-gtk3-run", + "moffy-samples-gtk4", + "moffy-samples-gtk4-run", + "mpi-hs", + "mpi-hs-binary", + "mpi-hs-cereal", + "mstate", + "mysql", + "mysql-json-table", + "mysql-simple", + "nanovg", + "netcode-io", + "nfc", + "NineP", + "nix-paths", + "nvvm", + "odbc", + "ods2csv", + "ogma-cli", + "opaleye", + "openssl-streams", + "OrderedBits", + "pagure-cli", + "pandoc-cli", + "parser-combinators-tests", + "pcre-heavy", + "pcre-light", + "peregrin", + "perf", + "persistent-mysql", + "persistent-postgresql", + "pg-transact", + "pkgtreediff", + "place-cursor-at", + "pontarius-xmpp", + "pontarius-xmpp-extras", + "postgresql-libpq-notify", + "postgresql-migration", + "postgresql-schema", + "postgresql-simple", + "postgresql-simple-url", + "primecount", + "profiterole", + "proto-lens-protobuf-types", + "psql-helpers", + "pthread", + "pulse-simple", + "rawfilepath", + "rdtsc", + "re2", + "reactive-balsa", + "reactive-jack", + "reanimate-svg", + "regex-pcre", + "rel8", + "resistor-cube", + "rex", + "rhbzquery", + "rocksdb-haskell", + "rocksdb-haskell-jprupp", + "rocksdb-query", + "rts", + "scrypt", + "sdl2", + "sdl2-gfx", + "sdl2-image", + "sdl2-mixer", + "sdl2-ttf", + "secp256k1-haskell", + "seqalign", + "servant-http-streams", + "servius", + "ses-html", + "shelltestrunner", + "snappy", + "sound-collage", + "soxlib", + "sphinx", + "split-record", + "sqlcli", + "sqlcli-odbc", + "sqlite-simple", + "stakhanov", // hasql + "stack-all", + "stack-clean-old", + "stack-templatizer", + "stringprep", + "SVGFonts", + "swizzle", + "swizzle-lens", + "swizzle-modify", + "swizzle-set", + "sydtest-persistent-postgresql", + "synthesizer-alsa", + "taffybar", + "tasty-papi", + "termonad", + "test-certs", + "text-icu", + "text-regex-replace", + "tls-debug", + "tmp-postgres", + "tmp-proc-postgres", + "ua-parser", + "uniq-deep", + "unix", + "users-postgresql-simple", + "validate-input", + "vector-fftw", + "visualize-type-inference", + "wai-session-postgresql", + "web3-tools", + "wild-bind-x11", + "Win32", + "Win32-notify", + "windns", + "X11", + "X11-xft", + "x11-xim", + "xmonad", + "xmonad-contrib", + "xmonad-extras", + "yesod-bin", + "yi", + "yi-frontend-pango", + "yoga", + "youtube", + "zeromq4-haskell", + "zeromq4-patterns", + "zot", + "ztail" + ], + "linux": [ + "hfsevents", + "vty-windows" + ], + "osx": [ + "vty-windows" + ], + "windows": [ + "hfsevents", + "postgresql-libpq", + "vty-unix" + ] + }, + // "Excluded pinned" are packages like excluded -- i.e. also excluded from + // build-depends -- but we nevertheless want pinned, for reproducibility. + // This makes sense for transitive dependencies that we cannot build + // directly e.g. exes used in setup. + "excluded_pinned": [ + "alex", + "happy" + ], + // "Unpinned" are packages that we /do/ want to build (i.e. not part of + // the "excluded" group) but do not want to pin exactly e.g. boot packages, + // because exact pins would prevent building with multiple GHCs. + // + // Boot packages taken from: + // https://gitlab.haskell.org/ghc/ghc/-/wikis/commentary/libraries/version-history + "unpinned": [ + "array", + "base", + "binary", + "bytestring", + "Cabal", + "Cabal-syntax", + "containers", + "deepseq", + "directory", + "exceptions", + "file-io", + "filepath", + "haddock-api", + "haddock-library", + "haskeline", + "hpc", + "integer-gmp", + "mtl", + "os-string", + "parsec", + "pretty", + "process", + "semaphore-compat", + "stm", + "template-haskell", + "terminfo", + "text", + "time", + "transformers", + "xhtml" + ] +} diff --git a/src/CLC/Stackage/Builder/Writer.hs b/src/CLC/Stackage/Builder/Writer.hs index 48787ea..1b87f4a 100644 --- a/src/CLC/Stackage/Builder/Writer.hs +++ b/src/CLC/Stackage/Builder/Writer.hs @@ -5,11 +5,12 @@ module CLC.Stackage.Builder.Writer where import CLC.Stackage.Builder.Batch (PackageGroup (unPackageGroup)) +import CLC.Stackage.Parser (PackageSet (extraPins, packageList, unpinned)) import CLC.Stackage.Utils.IO qualified as IO -import CLC.Stackage.Utils.Package (Package) import CLC.Stackage.Utils.Package qualified as Package import CLC.Stackage.Utils.Paths qualified as Paths import Data.List.NonEmpty qualified as NE +import Data.Set qualified as Set import Data.Text (Text) import Data.Text qualified as T import Data.Text.Encoding qualified as TEnc @@ -27,13 +28,32 @@ import Data.Text.Encoding qualified as TEnc -- By writing the entire (exact) dependency set into the cabal.project.local's -- constraints section, we ensure the same version of aeson is used every time -- it is a (transitive) dependency. -writeCabalProjectLocal :: [Package] -> IO () -writeCabalProjectLocal pkgs = IO.writeBinaryFile path constraintsSrc +writeCabalProjectLocal :: PackageSet -> IO () +writeCabalProjectLocal packageSet = IO.writeBinaryFile path constraintsSrc where path = Paths.generatedCabalProjectLocalPath constraintsSrc = TEnc.encodeUtf8 constraintsTxt constraintsTxt = T.unlines $ "constraints:" : constraints - constraints = (\p -> " " <> Package.toCabalConstraintsText p) <$> pkgs + -- Use any prefix so that this applies to setup too e.g. happy + constraints = (\p -> " any." <> Package.toCabalConstraintsText p) <$> constrainedPkgs + + -- In addition to all of the normal packages whose constraints we want to + -- write, we also need to: + -- + -- - Add in extraPins. + -- - Remove any unpinned. + constrainedPkgs = + Set.toList + . removeUnpinned + . addExtraPins + . Set.fromList + $ packageSet.packageList + + addExtraPins = Set.union (Set.fromList packageSet.extraPins) + + removeUnpinned = + Set.filter + (\p -> Set.notMember p.name (Set.fromList packageSet.unpinned)) -- | Writes the package set to a cabal file for building. This will be called -- for each group we want to build. diff --git a/src/CLC/Stackage/Parser.hs b/src/CLC/Stackage/Parser.hs index bc0669e..b652589 100644 --- a/src/CLC/Stackage/Parser.hs +++ b/src/CLC/Stackage/Parser.hs @@ -3,7 +3,9 @@ module CLC.Stackage.Parser ( -- * Retrieving packages + PackageSet (..), getPackageList, + packageListToSet, -- * Misc helpers printPackageList, @@ -25,11 +27,11 @@ import CLC.Stackage.Utils.OS (Os (Linux, Osx, Windows)) import CLC.Stackage.Utils.OS qualified as OS import CLC.Stackage.Utils.Package (Package) import CLC.Stackage.Utils.Package qualified as Package -import Control.Monad (when) +import Control.Exception (throwIO) +import Control.Monad (unless, when) import Data.Aeson (FromJSON, ToJSON) import Data.Foldable (for_) import Data.Maybe (fromMaybe) -import Data.Set (Set) import Data.Set qualified as Set import Data.Text (Text) import Data.Text qualified as T @@ -41,12 +43,18 @@ import System.OsPath (OsPath, osp) import System.Process qualified as P import Text.ParserCombinators.ReadP qualified as ReadP --- | Retrieves the list of packages, based on +-- | Retrieves the 'PackageSet', based on -- 'CLC.Stackage.Parser.API.stackageUrl'. -getPackageList :: Logging.Handle -> Maybe OsPath -> IO [Package] +getPackageList :: Logging.Handle -> Maybe OsPath -> IO PackageSet getPackageList hLogger msnapshotPath = do response <- getStackageResponse hLogger msnapshotPath - getPackageListByOs hLogger response OS.currentOs + packageListToSet hLogger response.packages + +-- | Given a list of packages, returns a 'PackageSet' i.e. all constraints +-- and filtered packages, according to package_index.jsonc +packageListToSet :: Logging.Handle -> [Package] -> IO PackageSet +packageListToSet hLogger packages = + getPackageListByOs hLogger packages OS.currentOs -- | Prints the package list to a file. printPackageList :: Logging.Handle -> Maybe OsPath -> Maybe Os -> IO () @@ -68,15 +76,41 @@ printPackageList hLogger msnapshotPath mOs = do -- | Retrieves the package list formatted to text. getPackageListByOsFmt :: Logging.Handle -> StackageResponse -> Os -> IO [Text] getPackageListByOsFmt hLogger response os = do - ps <- getPackageListByOs hLogger response os - pure $ Package.toDisplayName <$> ps + ps <- getPackageListByOs hLogger response.packages os + pure $ Package.toDisplayName <$> ps.packageList + +-- | PackageSet corresponds to the package set after filtering i.e. we +-- get some list of packages either from Stackage or --snapshot-path, +-- then we apply filtering according to package_index.jsonc +data PackageSet = MkPackageSet + { -- | Extra packages that are not in packageList, but nevertheless we want + -- to pin e.g. because they are transitive dependencies. + extraPins :: [Package], + -- | Package set after filtering out excluded packages. + packageList :: [Package], + -- | Package in packageList that we do not want to pin. + unpinned :: [Text] + } + deriving stock (Eq, Show) -- | Helper in case we want to see what the package set for a given OS is. -getPackageListByOs :: Logging.Handle -> StackageResponse -> Os -> IO [Package] -getPackageListByOs hLogger response os = do - excludedPkgs <- getExcludedPkgs os - let filterExcluded = flip Set.notMember excludedPkgs . (.name) - packages = filter filterExcluded response.packages +getPackageListByOs :: Logging.Handle -> [Package] -> Os -> IO PackageSet +getPackageListByOs hLogger packageList os = do + packageIndex <- getPackageIndex hLogger + + -- Setup extra pins i.e. all p in packageList in packageIndex.excluded_pinned. + let excludedPinnedCfg = Set.fromList packageIndex.excluded_pinned + isExcludedPinned = flip Set.member excludedPinnedCfg . (.name) + excludedPinned = filter isExcludedPinned packageList + + -- Setup excluded. This is packageIndex.excluded + + -- packageIndex.excluded_pinned. + let excluded = + Set.fromList (packageIndex.excluded.all ++ osSel packageIndex.excluded) + `Set.union` excludedPinnedCfg + + isNotExcluded = flip Set.notMember excluded . (.name) + packages = filter isNotExcluded packageList msg = mconcat [ "Filtered to ", @@ -87,7 +121,18 @@ getPackageListByOs hLogger response os = do ] Logging.putTimeInfoStr hLogger msg - pure packages + pure $ + MkPackageSet + { extraPins = excludedPinned, + packageList = packages, + unpinned = packageIndex.unpinned + } + where + osSel :: Excluded -> [Text] + osSel = case os of + Linux -> (.linux) + Osx -> (.osx) + Windows -> (.windows) getStackageResponse :: Logging.Handle -> Maybe OsPath -> IO StackageResponse getStackageResponse hLogger msnapshotPath = do @@ -123,28 +168,108 @@ getStackageResponse hLogger msnapshotPath = do pure response -getExcludedPkgs :: Os -> IO (Set Text) -getExcludedPkgs os = do +getPackageIndex :: Logging.Handle -> IO PackageIndex +getPackageIndex hLogger = do contents <- Ex.throwLeft . Utils.stripComments =<< IO.readBinaryFile path - excluded <- case JSON.decode contents of + packageIndex <- case JSON.decode @PackageIndex contents of Left err -> fail err Right x -> pure x - pure $ Set.fromList (excluded.all ++ osSel excluded) + verifyIndex packageIndex + + pure packageIndex where - path = [osp|excluded_pkgs.jsonc|] + path = [osp|package_index.jsonc|] - osSel :: Excluded -> [Text] - osSel = case os of - Linux -> (.linux) - Osx -> (.osx) - Windows -> (.windows) + -- Verifies index properties. + verifyIndex index = do + -- Each key should not have duplicates. A violation is mostly harmless, + -- presumably an oversight. + warnDuplicates "excluded.all" index.excluded.all + warnDuplicates "excluded.linux" index.excluded.linux + warnDuplicates "excluded.osx" index.excluded.osx + warnDuplicates "excluded.windows" index.excluded.windows + + warnDuplicates "excluded_pinned" index.excluded_pinned + warnDuplicates "unpinned" index.unpinned + + let allOs = Set.fromList index.excluded.all + linux = Set.fromList index.excluded.linux + osx = Set.fromList index.excluded.osx + windows = Set.fromList index.excluded.windows + + excluded = Set.unions [allOs, linux, osx, windows] + excludedPinned = Set.fromList index.excluded_pinned + unpinned = Set.fromList index.unpinned + + -- Each of these should be disjoint. Note that various excluded subkeys + -- may not be disjoint e.g. excluded.linux and excluded.osx both + -- exclude vty-windows. Hence we only check the combined excluded + -- against the other two. + -- + -- This is more serious than duplicate violations as it could lead + -- to confusing behavior e.g. if a package is in excluded_pinned + -- and unpinned. + errNonDisjoint "excluded" excluded "excluded_pinned" excludedPinned + errNonDisjoint "excluded" excluded "unpinned" unpinned + errNonDisjoint "excluded_pinned" excludedPinned "unpinned" unpinned + + errNonDisjoint xName x yName y = do + let z = Set.intersection x y + unless (Set.null z) $ do + let libs = T.intercalate ", " $ Set.toList z + msg = + mconcat + [ "package_index.jsonc: ", + xName, + " and ", + yName, + " are not disjoint: ", + libs + ] + Logging.putTimeErrStr hLogger msg + throwIO $ ExitFailure 1 + + warnDuplicates xName x = do + let (_, duplicates) = foldl' go (Set.empty, Set.empty) x + go (foundSoFar, dupes) l = + if Set.member l foundSoFar + then (foundSoFar, Set.insert l dupes) + else (Set.insert l foundSoFar, dupes) + + unless (Set.null duplicates) $ do + let libs = T.intercalate ", " $ Set.toList duplicates + msg = + mconcat + [ "package_index.jsonc: ", + xName, + " has duplicates: ", + libs + ] + Logging.putTimeWarnStr hLogger msg + +-- | Corresponds to package_index.jsonc +data PackageIndex = MkPackageIndex + { -- | Excluded packages. + excluded :: Excluded, + -- | Excluded packages that we nevertheless pin. + excluded_pinned :: [Text], + -- | Normal packages that should not be pinned. + unpinned :: [Text] + } + deriving stock (Eq, Generic, Show) + deriving anyclass (FromJSON, ToJSON) +-- | package_index.excluded. data Excluded = MkExcluded - { all :: [Text], + { -- | Packages excluded from all Os's. + all :: [Text], + -- | Packages excluded from linux. linux :: [Text], + -- | Packages excluded from osx. osx :: [Text], + -- | Packages excluded from windows. windows :: [Text] } deriving stock (Eq, Generic, Show) diff --git a/src/CLC/Stackage/Runner/Args.hs b/src/CLC/Stackage/Runner/Args.hs index 38ae88b..797fae1 100644 --- a/src/CLC/Stackage/Runner/Args.hs +++ b/src/CLC/Stackage/Runner/Args.hs @@ -480,7 +480,7 @@ parseSnapshotPath = "https://www.stackage.org//cabal.config i.e. each ", "line should be ' ==' e.g. 'lens ==5.3.4'. Note ", "that the snapshot is still filtered according to ", - "excluded_pkgs.jsonc." + "package_index.jsonc." ] ] ) diff --git a/src/CLC/Stackage/Runner/Env.hs b/src/CLC/Stackage/Runner/Env.hs index 6fa7a53..2def88a 100644 --- a/src/CLC/Stackage/Runner/Env.hs +++ b/src/CLC/Stackage/Runner/Env.hs @@ -28,6 +28,7 @@ import CLC.Stackage.Builder.Env ), ) import CLC.Stackage.Builder.Env qualified as Builder.Env +import CLC.Stackage.Parser (PackageSet (packageList)) import CLC.Stackage.Parser qualified as Parser import CLC.Stackage.Runner.Args ( Args (snapshotPath), @@ -43,7 +44,7 @@ import CLC.Stackage.Runner.Report qualified as Report import CLC.Stackage.Utils.Exception qualified as Ex import CLC.Stackage.Utils.IO qualified as IO import CLC.Stackage.Utils.Logging qualified as Logging -import CLC.Stackage.Utils.Package (Package (MkPackage, name, version)) +import CLC.Stackage.Utils.Package (Package) import CLC.Stackage.Utils.Paths qualified as Paths import Control.Exception (throwIO) import Control.Monad (join, when) @@ -61,7 +62,7 @@ import System.Directory.OsPath qualified as Dir import System.Exit (ExitCode (ExitSuccess)) import System.OsPath (osp) import System.OsPath qualified as OsP -import Prelude (IO, Monad ((>>=)), mconcat, pure, show, ($), (.), (<$>), (<>)) +import Prelude (IO, Monad ((>>=)), mconcat, pure, show, ($), (.), (<>)) -- | Args used for building all packages. data RunnerEnv = MkRunnerEnv @@ -80,7 +81,7 @@ data RunnerEnv = MkRunnerEnv -- | The complete package set from stackage. This is used to write the -- cabal.project.local's constraint section, to ensure we always use the -- same transitive dependencies. - completePackageSet :: [Package], + completePackageSet :: PackageSet, -- | Whether to retry packages that failed. retryFailures :: Bool, -- | Start time. @@ -144,10 +145,9 @@ setup hLoggerRaw modifyPackages = do (completePackageSet, pkgsList) <- case cache of Nothing -> do -- if no cache exists, query stackage - pkgsResponses <- Parser.getPackageList hLogger cliArgs.snapshotPath - let completePackageSet = responseToPkgs <$> pkgsResponses - pkgs = modifyPackages completePackageSet - pure (completePackageSet, pkgs) + packageSet <- Parser.getPackageList hLogger cliArgs.snapshotPath + let pkgs = modifyPackages packageSet.packageList + pure (packageSet, pkgs) Just oldResults -> do -- cache exists, use it rather than stackage oldFailures <- @@ -164,7 +164,11 @@ setup hLoggerRaw modifyPackages = do untested = oldResults.untested toTest = Set.union untested oldFailures - pure (Set.toList completePackageSet, Set.toList toTest) + -- Even though we read the packages from the cache, we still want to + -- get the packageSet, for writing the cabal.project.local constraints. + packageSet <- Parser.packageListToSet hLogger (Set.toList completePackageSet) + + pure (packageSet, Set.toList toTest) packagesToBuild <- case pkgsList of (p : ps) -> pure (p :| ps) @@ -206,12 +210,6 @@ setup hLoggerRaw modifyPackages = do retryFailures = cliArgs.retryFailures, startTime } - where - responseToPkgs p = - MkPackage - { name = p.name, - version = p.version - } -- | Prints summary and writes results to disk. teardown :: RunnerEnv -> IO () diff --git a/test/unit/Unit/Prelude.hs b/test/unit/Unit/Prelude.hs index edfed4e..3f7ba77 100644 --- a/test/unit/Unit/Prelude.hs +++ b/test/unit/Unit/Prelude.hs @@ -20,6 +20,14 @@ import CLC.Stackage.Builder.Env ), Progress (MkProgress, failuresRef, successesRef), ) +import CLC.Stackage.Parser + ( PackageSet + ( MkPackageSet, + extraPins, + packageList, + unpinned + ), + ) import CLC.Stackage.Runner.Env ( RunnerEnv ( MkRunnerEnv, @@ -51,7 +59,12 @@ mkRunnerEnv = do MkRunnerEnv { buildEnv, cache = Nothing, - completePackageSet = NE.toList buildEnv.packagesToBuild, + completePackageSet = + MkPackageSet + { extraPins = [], + packageList = NE.toList buildEnv.packagesToBuild, + unpinned = [] + }, cabalUpdate = False, cacheEnabled = True, cleanup = True,