ENT-13005: Added integrity check to cf-remote remote installation

MANIFEST.in

@@ -2,3 +2,5 @@ include cf_remote/nt-discovery.sh
 include cf_remote/Vagrantfile
 include cf_remote/default_provision.sh
 include cf_remote/demo.sql
+include cf_remote/remote-download.sh
+

cf_remote/remote-download.sh

@@ -0,0 +1,89 @@
+#!/bin/bash
+
+set -e
+
+# Arg parsing
+
+INSECURE=0
+CHECKSUM=""
+
+while getopts c:I option; do
+    case "${option}" in
+    I)
+        INSECURE=1
+        ;;
+    c)
+        CHECKSUM=$OPTARG
+        ;;
+    *)
+      echo "Usage: $0 [-I] [-c checksum] package_url"
+      exit 1
+      ;;
+    esac
+done
+
+shift $((OPTIND - 1))
+
+PACKAGE=$1
+if [ -z "$PACKAGE" ]; then
+  echo "Usage: $0 [-I] [-c checksum] package_url"
+  exit 1;
+fi
+
+# checks
+
+HAS_SHA256SUM="$(command -v sha256sum)"
+
+if [ -z "$HAS_SHA256SUM" ]; then
+    if [ "$INSECURE" -eq 0 ]; then
+        echo "Cannot check file integrity. sha256sum is not installed on host. Run with --insecure to skip"
+        exit 1
+    fi
+    echo "Cannot check file integrity. sha256sum is not installed on host. Continuing due to insecure flag"
+fi
+
+if [ -z "$CHECKSUM" ]; then
+    if [ "$INSECURE" -eq 0 ]; then
+        echo "Cannot check file integrity. No artifact associated with package '$PACKAGE' found. Run with --insecure to skip"
+        exit 1
+    fi
+    echo "Cannot check file integrity. No artifact associated with package '$PACKAGE' found. Continuing due to insecure flag"
+fi
+
+# temp file
+
+tmpfile=$(mktemp)
+cleanup() {
+  rm -f "$tmpfile"
+}
+trap cleanup EXIT QUIT TERM
+
+# Download
+
+if [ -n "$(command -v wget)" ]; then
+    wget -nv -O "$tmpfile" "$PACKAGE"
+elif [ -n "$(command -v curl)" ]; then
+    curl --fail -sS -o "$tmpfile" "$PACKAGE"
+else
+    echo "Cannot download remotely. wget and/or curl are not installed on host"
+    exit 1
+fi
+
+# Checksum
+
+filename="$(basename "$PACKAGE")"
+
+if [ -n "$HAS_SHA256SUM" ] && [ -n "$CHECKSUM" ]; then
+    hash="$(sha256sum "$tmpfile" | cut -d ' ' -f 1)"
+
+    if [[ "$CHECKSUM" != "$hash" ]]; then
+        if [ "$INSECURE" -eq 0 ]; then
+          echo "Package '$PACKAGE' doesn't match the expected checksum '$CHECKSUM'. Run with --insecure to skip"
+          exit 1
+        fi
+        echo "Package '$PACKAGE' doesn't match the expected checksum '$CHECKSUM'. Continuing due to insecure flag"
+    fi
+fi
+
+mv "$tmpfile" "$filename"
+

cf_remote/remote.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 import sys
 import re
+
 from os.path import basename, dirname, join, exists
 from collections import OrderedDict
 from typing import Union
@@ -276,7 +277,17 @@ def get_info(host, *, users=None, connection=None):
         data["role"] = "hub" if discovery.get("NTD_CFHUB") else "client"
 
         data["bin"] = {}
-        for bin in ["dpkg", "rpm", "yum", "apt", "pkg", "zypper", "curl"]:
+        for bin in [
+            "dpkg",
+            "rpm",
+            "yum",
+            "apt",
+            "pkg",
+            "zypper",
+            "curl",
+            "wget",
+            "sha256sum",
+        ]:
             path = discovery.get("NTD_{}".format(bin.upper()))
             if path:
                 data["bin"][bin] = path
@@ -432,7 +443,7 @@ def bootstrap_host(host_data, policy_server, *, connection=None, trust_server=Tr
 def _package_from_list(tags, extension, packages):
     artifacts = [Artifact(None, p) for p in packages]
     artifact = filter_artifacts(artifacts, tags, extension)[-1]
-    return artifact.url
+    return artifact.url, artifact
 
 
 def _package_from_releases(
@@ -445,7 +456,7 @@ def _package_from_releases(
         release = releases.pick_version(version)
     if release is None:
         print("Could not find a release for version {}".format(version))
-        return None
+        return None, None
 
     release.init_download()
 
@@ -455,7 +466,7 @@ def _package_from_releases(
                 version, edition
             )
         )
-        return None
+        return None, None
 
     artifacts = release.find(tags, extension)
     if not artifacts:
@@ -464,13 +475,16 @@ def _package_from_releases(
                 "hub" if "hub" in tags else "client"
             )
         )
-        return None
+        return None, None
     artifact = artifacts[-1]
     if remote_download:
-        return artifact.url
+        return artifact.url, artifact
     else:
-        return download_package(
-            artifact.url, checksum=artifact.checksum, insecure=insecure
+        return (
+            download_package(
+                artifact.url, checksum=artifact.checksum, insecure=insecure
+            ),
+            artifact,
         )
 
 
@@ -514,13 +528,44 @@ def get_package_from_host_info(
         tags.extend(tag for tag in package_tags if tag != "msi")
 
     if packages is None:  # No command line argument given
-        package = _package_from_releases(
+        package, artifact = _package_from_releases(
             tags, extension, version, edition, remote_download, insecure
         )
     else:
-        package = _package_from_list(tags, extension, packages)
+        package, artifact = _package_from_list(tags, extension, packages)
+
+    return package, artifact
+
+
+def _remote_download(host, package, artifact, insecure=False, connection=None):
+
+    cf_remote_dir = dirname(__file__)
+    script_path = join(cf_remote_dir, "remote-download.sh")
+    if not exists(script_path):
+        sys.exit("%s does not exist" % script_path)
+    scp(
+        script_path,
+        host,
+        connection,
+        hide=True,
+    )
+    args = ""
+    if insecure:
+        args += "-I "
+    if artifact:
+        args += "-c {} ".format(artifact.checksum)
+    args += package
 
-    return package
+    ret = ssh_cmd(connection, "bash remote-download.sh {}".format(args), errors=True)
+
+    if ret is None:
+        return None
+    if insecure:
+        log.warning(ret)
+
+    log.debug("Successfully remotely installed package on host")
+
+    return basename(package)
 
 
 @auto_connect
@@ -552,9 +597,10 @@ def install_host(
     elif packages and len(packages) == 1:
         package = packages[0]
 
+    artifact = None
     if not package:
         try:
-            package = get_package_from_host_info(
+            package, artifact = get_package_from_host_info(
                 data.get("package_tags"),
                 data.get("bin"),
                 data.get("arch"),
@@ -574,19 +620,11 @@ def install_host(
         return 1
 
     if remote_download:
-        if ("bin" not in data) or ("curl" not in data["bin"]):
-            log.error(
-                "Couldn't download remotely. Curl is not installed on host '%s'" % host
-            )
-            return 1
-
-        print("Downloading '%s' on '%s' using curl" % (package, host))
-        r = ssh_cmd(
-            cmd="curl --fail -O {}".format(package), connection=connection, errors=True
+        package = _remote_download(
+            host, package, artifact, connection=connection, insecure=insecure
         )
-        if r is None:
+        if package is None:
             return 1
-        package = basename(package)
     elif not getattr(connection, "is_local", False):
         scp(package, host, connection=connection)
         package = basename(package)