diff --git a/exports/taskfiles/utils/checksum.yaml b/exports/taskfiles/utils/checksum.yaml index 8896296..65499ab 100644 --- a/exports/taskfiles/utils/checksum.yaml +++ b/exports/taskfiles/utils/checksum.yaml @@ -10,8 +10,8 @@ tasks: # # @param {string} CHECKSUM_FILE # @param {string[]} INCLUDE_PATTERNS Path wildcard patterns to compute the checksum for. - # @param {string[]} [EXCLUDE_PATTERNS=[]] Path wildcard patterns, relative to any - # `INCLUDE_PATTERNS`, to exclude from the checksum. + # @param {string[]} [EXCLUDE_PATTERNS=[]] Path wildcard patterns to exclude from checksum + # computation, applied as glob-style matches to full paths rooted at the `INCLUDE_PATTERNS`. # @param {string} [IGNORE_ERROR="false"] If set to "true", the task will not fail on error. compute: internal: true @@ -75,8 +75,8 @@ tasks: # # @param {string} CHECKSUM_FILE # @param {string[]} INCLUDE_PATTERNS Path wildcard patterns to validate the checksum for. - # @param {string[]} [EXCLUDE_PATTERNS=[]] Path wildcard patterns, relative to any - # `INCLUDE_PATTERNS`, to exclude from the checksum. + # @param {string[]} [EXCLUDE_PATTERNS=[]] Path wildcard patterns to exclude from checksum + # computation, applied as glob-style matches to full paths rooted at the `INCLUDE_PATTERNS`. # @param {string} [FAIL_ON_ERROR="false"] If set to "true", the task fails when checksums # mismatch. validate: diff --git a/exports/taskfiles/utils/remote.yaml b/exports/taskfiles/utils/remote.yaml index 0a99767..44c5dc2 100644 --- a/exports/taskfiles/utils/remote.yaml +++ b/exports/taskfiles/utils/remote.yaml @@ -60,6 +60,11 @@ tasks: # downloaded tar file. # @param {string[]} [EXCLUDE_PATTERNS] Path wildcard patterns that should not be extracted. # @param {string[]} [INCLUDE_PATTERNS] Path wildcard patterns to extract. + # @param {string[]} [CHECKSUM_EXCLUDE_PATTERNS=[]] Path wildcard patterns to exclude from checksum + # computation, applied as glob-style matches to full paths rooted at the + # `CHECKSUM_INCLUDE_PATTERNS`. + # @param {string[]} [CHECKSUM_INCLUDE_PATTERNS=[{{.OUTPUT_DIR}}]] Path wildcard patterns to + # compute the checksum for. # @param {int} [NUM_COMPONENTS_TO_STRIP=1] Number of leading path components to strip from the # extracted files. # @param {string} [TAR_FILE={{.OUTPUT_DIR}}.tar.gz] Path where the tar file should be stored. @@ -79,6 +84,12 @@ tasks: INCLUDE_PATTERNS: ref: "default (list) .INCLUDE_PATTERNS" + # Checksum path patterns + CHECKSUM_EXCLUDE_PATTERNS: + ref: "default (list) .CHECKSUM_EXCLUDE_PATTERNS" + CHECKSUM_INCLUDE_PATTERNS: + ref: "default (list .OUTPUT_DIR) .CHECKSUM_INCLUDE_PATTERNS" + # Misc ARCHIVER: >- {{if eq OS "darwin"}}gtar{{else}}tar{{end}} @@ -96,7 +107,10 @@ tasks: - task: "checksum:validate" vars: CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" - INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"] + EXCLUDE_PATTERNS: + ref: ".CHECKSUM_EXCLUDE_PATTERNS" + INCLUDE_PATTERNS: + ref: ".CHECKSUM_INCLUDE_PATTERNS" cmds: - |- rm -rf "{{.OUTPUT_DIR}}" @@ -122,7 +136,10 @@ tasks: - task: "checksum:compute" vars: CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" - INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"] + EXCLUDE_PATTERNS: + ref: ".CHECKSUM_EXCLUDE_PATTERNS" + INCLUDE_PATTERNS: + ref: ".CHECKSUM_INCLUDE_PATTERNS" # Uses curl to download a zip file from the given URL and extracts its contents. # @@ -133,6 +150,11 @@ tasks: # downloaded zip file. # @param {string[]} [EXCLUDE_PATTERNS] Path wildcard patterns that should not be extracted. # @param {string[]} [INCLUDE_PATTERNS] Path wildcard patterns to extract. + # @param {string[]} [CHECKSUM_EXCLUDE_PATTERNS=[]] Path wildcard patterns to exclude from checksum + # computation, applied as glob-style matches to full paths rooted at the + # `CHECKSUM_INCLUDE_PATTERNS`. + # @param {string[]} [CHECKSUM_INCLUDE_PATTERNS=[{{.OUTPUT_DIR}}]] Path wildcard patterns to + # compute the checksum for. # @param {string} [ZIP_FILE={{.OUTPUT_DIR}}.zip] Path where the zip file should be stored. download-and-extract-zip: internal: true @@ -150,6 +172,11 @@ tasks: INCLUDE_PATTERNS: ref: "default (list) .INCLUDE_PATTERNS" + # Checksum path patterns + CHECKSUM_EXCLUDE_PATTERNS: + ref: "default (list) .CHECKSUM_EXCLUDE_PATTERNS" + CHECKSUM_INCLUDE_PATTERNS: + ref: "default (list .OUTPUT_DIR) .CHECKSUM_INCLUDE_PATTERNS" requires: vars: ["FILE_SHA256", "OUTPUT_DIR", "URL"] sources: ["{{.TASKFILE}}", "{{.ZIP_FILE}}"] @@ -163,7 +190,10 @@ tasks: - task: "checksum:validate" vars: CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" - INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"] + EXCLUDE_PATTERNS: + ref: ".CHECKSUM_EXCLUDE_PATTERNS" + INCLUDE_PATTERNS: + ref: ".CHECKSUM_INCLUDE_PATTERNS" cmds: - |- rm -rf "{{.OUTPUT_DIR}}" @@ -184,4 +214,7 @@ tasks: - task: "checksum:compute" vars: CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" - INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"] + EXCLUDE_PATTERNS: + ref: ".CHECKSUM_EXCLUDE_PATTERNS" + INCLUDE_PATTERNS: + ref: ".CHECKSUM_INCLUDE_PATTERNS" diff --git a/taskfiles/remote/tests.yaml b/taskfiles/remote/tests.yaml index 7d05dd0..cd9581a 100644 --- a/taskfiles/remote/tests.yaml +++ b/taskfiles/remote/tests.yaml @@ -26,6 +26,8 @@ tasks: - task: "download-and-extract-zip-test-basic" - task: "download-and-extract-zip-test-exclusions" - task: "download-and-extract-zip-test-inclusions" + - task: "download-and-extract-zip-test-checksum-exclude" + - task: "download-and-extract-zip-test-checksum-include" curl-test-success: vars: @@ -137,6 +139,101 @@ tasks: - "test -e '{{.OUTPUT_DIR}}/{{.G_EXTRACTED_ZIP_CODEOWNERS_PATH}}'" - "test -e '{{.OUTPUT_DIR}}/{{.G_EXTRACTED_ZIP_PULL_REQUEST_TEMPLATE_PATH}}'" + download-and-extract-zip-test-checksum-exclude: + vars: + OUTPUT_DIR: "{{.G_OUTPUT_DIR}}/{{.TASK | replace \":\" \"#\"}}" + CHECKSUM_FILE_0: "{{.OUTPUT_DIR}}.md5.0" + CHECKSUM_FILE_1: "{{.OUTPUT_DIR}}.md5.1" + cmds: + - defer: |- + rm -f "{{.CHECKSUM_FILE_0}}" + rm -f "{{.CHECKSUM_FILE_1}}" + - defer: + task: "remote-test-cleaner" + vars: + OUTPUT_DIR: "{{.OUTPUT_DIR}}" + + # Case 0: Extract all files, and compute the checksum while excluding a subset of extracted + # paths from the checksum. + - task: "remote-test-cleaner" + vars: + OUTPUT_DIR: "{{.OUTPUT_DIR}}" + - task: "remote:download-and-extract-zip" + vars: + CHECKSUM_EXCLUDE_PATTERNS: + - ".github/CODEOWNERS" + - ".github/PULL_REQUEST_TEMPLATE.md" + CHECKSUM_FILE: "{{.CHECKSUM_FILE_0}}" + OUTPUT_DIR: "{{.OUTPUT_DIR}}" + URL: "{{.G_TEST_ZIP_FILE_URL}}" + FILE_SHA256: "{{.G_TEST_ZIP_FILE_SHA256}}" + + # Case 1: Exclude the same paths during extraction, and compute the checksum of the whole + # extracted directory. + - task: "remote-test-cleaner" + vars: + OUTPUT_DIR: "{{.OUTPUT_DIR}}" + - task: "remote:download-and-extract-zip" + vars: + CHECKSUM_FILE: "{{.CHECKSUM_FILE_1}}" + EXCLUDE_PATTERNS: + - "*/CODEOWNERS" + - "{{.G_EXTRACTED_ZIP_PULL_REQUEST_TEMPLATE_PATH}}" + OUTPUT_DIR: "{{.OUTPUT_DIR}}" + URL: "{{.G_TEST_ZIP_FILE_URL}}" + FILE_SHA256: "{{.G_TEST_ZIP_FILE_SHA256}}" + + # Test that the checksums from the two cases match + - "cmp -s '{{.CHECKSUM_FILE_0}}' '{{.CHECKSUM_FILE_1}}'" + + download-and-extract-zip-test-checksum-include: + vars: + OUTPUT_DIR: "{{.G_OUTPUT_DIR}}/{{.TASK | replace \":\" \"#\"}}" + CHECKSUM_FILE: "{{.OUTPUT_DIR}}.md5" + CHECKSUM_MOD_TS: "{{.CHECKSUM_FILE}}-mod-ts.txt" + REMOVED_FILE: "{{.OUTPUT_DIR}}/{{.G_EXTRACTED_ZIP_LICENSE_PATH}}" + cmds: + - defer: |- + rm -f "{{.CHECKSUM_FILE}}" + rm -f "{{.CHECKSUM_MOD_TS}}" + - defer: + task: "remote-test-cleaner" + vars: + OUTPUT_DIR: "{{.OUTPUT_DIR}}" + + # Extract all files, and compute the checksum while including only a subset of extracted paths + # in the checksum. Record the checksum timestamp. + - task: "remote-test-cleaner" + vars: + OUTPUT_DIR: "{{.OUTPUT_DIR}}" + - task: "remote:download-and-extract-zip" + vars: + CHECKSUM_INCLUDE_PATTERNS: &checksum_include_patterns + - "{{.OUTPUT_DIR}}/{{.G_EXTRACTED_ZIP_CODEOWNERS_PATH}}" + - "{{.OUTPUT_DIR}}/{{.G_EXTRACTED_ZIP_PULL_REQUEST_TEMPLATE_PATH}}" + CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" + OUTPUT_DIR: "{{.OUTPUT_DIR}}" + URL: "{{.G_TEST_ZIP_FILE_URL}}" + FILE_SHA256: "{{.G_TEST_ZIP_FILE_SHA256}}" + - "date -r '{{.CHECKSUM_FILE}}' > '{{.CHECKSUM_MOD_TS}}'" + + # Remove a file outside the checksum include set, rerun without cleaning, and verify the task + # skips because the included checksum scope is unchanged. + - "rm -f '{{.REMOVED_FILE}}'" + - task: "remote:download-and-extract-zip" + vars: + CHECKSUM_INCLUDE_PATTERNS: *checksum_include_patterns + CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" + OUTPUT_DIR: "{{.OUTPUT_DIR}}" + URL: "{{.G_TEST_ZIP_FILE_URL}}" + FILE_SHA256: "{{.G_TEST_ZIP_FILE_SHA256}}" + + # Test that the task skipped on the second call: + # - The removed file was not restored. + # - The checksum timestamp has not changed. + - "test ! -e '{{.REMOVED_FILE}}'" + - "cmp -s '{{.CHECKSUM_MOD_TS}}' <(date -r '{{.CHECKSUM_FILE}}')" + # Cleans up the files output by remote tasks (assuming their default paths weren't changed). # # @param {string} OUTPUT_DIR Output directory passed to remote tasks.