feat(remote): Add checksum path filters for download and extract tasks (fixes #68).

exports/taskfiles/utils/checksum.yaml

@@ -10,8 +10,8 @@ tasks:
   #
   # @param {string} CHECKSUM_FILE
   # @param {string[]} INCLUDE_PATTERNS Path wildcard patterns to compute the checksum for.
-  # @param {string[]} [EXCLUDE_PATTERNS=[]] Path wildcard patterns, relative to any
-  # `INCLUDE_PATTERNS`, to exclude from the checksum.
+  # @param {string[]} [EXCLUDE_PATTERNS=[]] Path wildcard patterns to exclude from checksum
+  # computation, applied as glob-style matches to full paths rooted at the `INCLUDE_PATTERNS`.
   # @param {string} [IGNORE_ERROR="false"] If set to "true", the task will not fail on error.
   compute:
     internal: true
@@ -75,8 +75,8 @@ tasks:
   #
   # @param {string} CHECKSUM_FILE
   # @param {string[]} INCLUDE_PATTERNS Path wildcard patterns to validate the checksum for.
-  # @param {string[]} [EXCLUDE_PATTERNS=[]] Path wildcard patterns, relative to any
-  # `INCLUDE_PATTERNS`, to exclude from the checksum.
+  # @param {string[]} [EXCLUDE_PATTERNS=[]] Path wildcard patterns to exclude from checksum
+  # computation, applied as glob-style matches to full paths rooted at the `INCLUDE_PATTERNS`.
   # @param {string} [FAIL_ON_ERROR="false"] If set to "true", the task fails when checksums
   # mismatch.
   validate:

exports/taskfiles/utils/remote.yaml

@@ -60,6 +60,11 @@ tasks:
   # downloaded tar file.
   # @param {string[]} [EXCLUDE_PATTERNS] Path wildcard patterns that should not be extracted.
   # @param {string[]} [INCLUDE_PATTERNS] Path wildcard patterns to extract.
+  # @param {string[]} [CHECKSUM_EXCLUDE_PATTERNS=[]] Path wildcard patterns to exclude from checksum
+  # computation, applied as glob-style matches to full paths rooted at the
+  # `CHECKSUM_INCLUDE_PATTERNS`.
+  # @param {string[]} [CHECKSUM_INCLUDE_PATTERNS=[{{.OUTPUT_DIR}}]] Path wildcard patterns to
+  # compute the checksum for.
   # @param {int} [NUM_COMPONENTS_TO_STRIP=1] Number of leading path components to strip from the
   # extracted files.
   # @param {string} [TAR_FILE={{.OUTPUT_DIR}}.tar.gz] Path where the tar file should be stored.
@@ -79,6 +84,12 @@ tasks:
       INCLUDE_PATTERNS:
         ref: "default (list) .INCLUDE_PATTERNS"
 
+      # Checksum path patterns
+      CHECKSUM_EXCLUDE_PATTERNS:
+        ref: "default (list) .CHECKSUM_EXCLUDE_PATTERNS"
+      CHECKSUM_INCLUDE_PATTERNS:
+        ref: "default (list .OUTPUT_DIR) .CHECKSUM_INCLUDE_PATTERNS"
+
       # Misc
       ARCHIVER: >-
         {{if eq OS "darwin"}}gtar{{else}}tar{{end}}
@@ -96,7 +107,10 @@ tasks:
       - task: "checksum:validate"
         vars:
           CHECKSUM_FILE: "{{.CHECKSUM_FILE}}"
-          INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"]
+          EXCLUDE_PATTERNS:
+            ref: ".CHECKSUM_EXCLUDE_PATTERNS"
+          INCLUDE_PATTERNS:
+            ref: ".CHECKSUM_INCLUDE_PATTERNS"
     cmds:
       - |-
         rm -rf "{{.OUTPUT_DIR}}"
@@ -122,7 +136,10 @@ tasks:
       - task: "checksum:compute"
         vars:
           CHECKSUM_FILE: "{{.CHECKSUM_FILE}}"
-          INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"]
+          EXCLUDE_PATTERNS:
+            ref: ".CHECKSUM_EXCLUDE_PATTERNS"
+          INCLUDE_PATTERNS:
+            ref: ".CHECKSUM_INCLUDE_PATTERNS"
 
   # Uses curl to download a zip file from the given URL and extracts its contents.
   #
@@ -133,6 +150,11 @@ tasks:
   # downloaded zip file.
   # @param {string[]} [EXCLUDE_PATTERNS] Path wildcard patterns that should not be extracted.
   # @param {string[]} [INCLUDE_PATTERNS] Path wildcard patterns to extract.
+  # @param {string[]} [CHECKSUM_EXCLUDE_PATTERNS=[]] Path wildcard patterns to exclude from checksum
+  # computation, applied as glob-style matches to full paths rooted at the
+  # `CHECKSUM_INCLUDE_PATTERNS`.
+  # @param {string[]} [CHECKSUM_INCLUDE_PATTERNS=[{{.OUTPUT_DIR}}]] Path wildcard patterns to
+  # compute the checksum for.
   # @param {string} [ZIP_FILE={{.OUTPUT_DIR}}.zip] Path where the zip file should be stored.
   download-and-extract-zip:
     internal: true
@@ -150,6 +172,11 @@ tasks:
       INCLUDE_PATTERNS:
         ref: "default (list) .INCLUDE_PATTERNS"
 
+      # Checksum path patterns
+      CHECKSUM_EXCLUDE_PATTERNS:
+        ref: "default (list) .CHECKSUM_EXCLUDE_PATTERNS"
+      CHECKSUM_INCLUDE_PATTERNS:
+        ref: "default (list .OUTPUT_DIR) .CHECKSUM_INCLUDE_PATTERNS"
     requires:
       vars: ["FILE_SHA256", "OUTPUT_DIR", "URL"]
     sources: ["{{.TASKFILE}}", "{{.ZIP_FILE}}"]
@@ -163,7 +190,10 @@ tasks:
       - task: "checksum:validate"
         vars:
           CHECKSUM_FILE: "{{.CHECKSUM_FILE}}"
-          INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"]
+          EXCLUDE_PATTERNS:
+            ref: ".CHECKSUM_EXCLUDE_PATTERNS"
+          INCLUDE_PATTERNS:
+            ref: ".CHECKSUM_INCLUDE_PATTERNS"
     cmds:
       - |-
         rm -rf "{{.OUTPUT_DIR}}"
@@ -184,4 +214,7 @@ tasks:
       - task: "checksum:compute"
         vars:
           CHECKSUM_FILE: "{{.CHECKSUM_FILE}}"
-          INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"]
+          EXCLUDE_PATTERNS:
+            ref: ".CHECKSUM_EXCLUDE_PATTERNS"
+          INCLUDE_PATTERNS:
+            ref: ".CHECKSUM_INCLUDE_PATTERNS"

taskfiles/remote/tests.yaml

@@ -26,6 +26,8 @@ tasks:
       - task: "download-and-extract-zip-test-basic"
       - task: "download-and-extract-zip-test-exclusions"
       - task: "download-and-extract-zip-test-inclusions"
+      - task: "download-and-extract-zip-test-checksum-exclude"
+      - task: "download-and-extract-zip-test-checksum-include"
 
   curl-test-success:
     vars:
@@ -137,6 +139,101 @@ tasks:
       - "test -e '{{.OUTPUT_DIR}}/{{.G_EXTRACTED_ZIP_CODEOWNERS_PATH}}'"
       - "test -e '{{.OUTPUT_DIR}}/{{.G_EXTRACTED_ZIP_PULL_REQUEST_TEMPLATE_PATH}}'"
 
+  download-and-extract-zip-test-checksum-exclude:
+    vars:
+      OUTPUT_DIR: "{{.G_OUTPUT_DIR}}/{{.TASK | replace \":\" \"#\"}}"
+      CHECKSUM_FILE_0: "{{.OUTPUT_DIR}}.md5.0"
+      CHECKSUM_FILE_1: "{{.OUTPUT_DIR}}.md5.1"
+    cmds:
+      - defer: |-
+          rm -f "{{.CHECKSUM_FILE_0}}"
+          rm -f "{{.CHECKSUM_FILE_1}}"
+      - defer:
+          task: "remote-test-cleaner"
+          vars:
+            OUTPUT_DIR: "{{.OUTPUT_DIR}}"
+
+      # Case 0: Extract all files, and compute the checksum while excluding a subset of extracted
+      # paths from the checksum.
+      - task: "remote-test-cleaner"
+        vars:
+          OUTPUT_DIR: "{{.OUTPUT_DIR}}"
+      - task: "remote:download-and-extract-zip"
+        vars:
+          CHECKSUM_EXCLUDE_PATTERNS:
+            - ".github/CODEOWNERS"
+            - ".github/PULL_REQUEST_TEMPLATE.md"
+          CHECKSUM_FILE: "{{.CHECKSUM_FILE_0}}"
+          OUTPUT_DIR: "{{.OUTPUT_DIR}}"
+          URL: "{{.G_TEST_ZIP_FILE_URL}}"
+          FILE_SHA256: "{{.G_TEST_ZIP_FILE_SHA256}}"
+
+      # Case 1: Exclude the same paths during extraction, and compute the checksum of the whole
+      # extracted directory.
+      - task: "remote-test-cleaner"
+        vars:
+          OUTPUT_DIR: "{{.OUTPUT_DIR}}"
+      - task: "remote:download-and-extract-zip"
+        vars:
+          CHECKSUM_FILE: "{{.CHECKSUM_FILE_1}}"
+          EXCLUDE_PATTERNS:
+            - "*/CODEOWNERS"
+            - "{{.G_EXTRACTED_ZIP_PULL_REQUEST_TEMPLATE_PATH}}"
+          OUTPUT_DIR: "{{.OUTPUT_DIR}}"
+          URL: "{{.G_TEST_ZIP_FILE_URL}}"
+          FILE_SHA256: "{{.G_TEST_ZIP_FILE_SHA256}}"
+
+      # Test that the checksums from the two cases match
+      - "cmp -s '{{.CHECKSUM_FILE_0}}' '{{.CHECKSUM_FILE_1}}'"
+
+  download-and-extract-zip-test-checksum-include:
+    vars:
+      OUTPUT_DIR: "{{.G_OUTPUT_DIR}}/{{.TASK | replace \":\" \"#\"}}"
+      CHECKSUM_FILE: "{{.OUTPUT_DIR}}.md5"
+      CHECKSUM_MOD_TS: "{{.CHECKSUM_FILE}}-mod-ts.txt"
+      REMOVED_FILE: "{{.OUTPUT_DIR}}/{{.G_EXTRACTED_ZIP_LICENSE_PATH}}"
+    cmds:
+      - defer: |-
+          rm -f "{{.CHECKSUM_FILE}}"
+          rm -f "{{.CHECKSUM_MOD_TS}}"
+      - defer:
+          task: "remote-test-cleaner"
+          vars:
+            OUTPUT_DIR: "{{.OUTPUT_DIR}}"
+
+      # Extract all files, and compute the checksum while including only a subset of extracted paths
+      # in the checksum. Record the checksum timestamp.
+      - task: "remote-test-cleaner"
+        vars:
+          OUTPUT_DIR: "{{.OUTPUT_DIR}}"
+      - task: "remote:download-and-extract-zip"
+        vars:
+          CHECKSUM_INCLUDE_PATTERNS: &checksum_include_patterns
+            - "{{.OUTPUT_DIR}}/{{.G_EXTRACTED_ZIP_CODEOWNERS_PATH}}"
+            - "{{.OUTPUT_DIR}}/{{.G_EXTRACTED_ZIP_PULL_REQUEST_TEMPLATE_PATH}}"
+          CHECKSUM_FILE: "{{.CHECKSUM_FILE}}"
+          OUTPUT_DIR: "{{.OUTPUT_DIR}}"
+          URL: "{{.G_TEST_ZIP_FILE_URL}}"
+          FILE_SHA256: "{{.G_TEST_ZIP_FILE_SHA256}}"
+      - "date -r '{{.CHECKSUM_FILE}}' > '{{.CHECKSUM_MOD_TS}}'"
+
+      # Remove a file outside the checksum include set, rerun without cleaning, and verify the task
+      # skips because the included checksum scope is unchanged.
+      - "rm -f '{{.REMOVED_FILE}}'"
+      - task: "remote:download-and-extract-zip"
+        vars:
+          CHECKSUM_INCLUDE_PATTERNS: *checksum_include_patterns
+          CHECKSUM_FILE: "{{.CHECKSUM_FILE}}"
+          OUTPUT_DIR: "{{.OUTPUT_DIR}}"
+          URL: "{{.G_TEST_ZIP_FILE_URL}}"
+          FILE_SHA256: "{{.G_TEST_ZIP_FILE_SHA256}}"
+
+      # Test that the task skipped on the second call:
+      # - The removed file was not restored.
+      # - The checksum timestamp has not changed.
+      - "test ! -e '{{.REMOVED_FILE}}'"
+      - "cmp -s '{{.CHECKSUM_MOD_TS}}' <(date -r '{{.CHECKSUM_FILE}}')"
+
   # Cleans up the files output by remote tasks (assuming their default paths weren't changed).
   #
   # @param {string} OUTPUT_DIR Output directory passed to remote tasks.