diff --git a/make/scripts/generate-debian-repo.sh b/make/scripts/generate-debian-repo.sh new file mode 100755 index 000000000000..286433019be7 --- /dev/null +++ b/make/scripts/generate-debian-repo.sh @@ -0,0 +1,265 @@ +#!/usr/bin/env bash +# Copyright (c) 2024 SAP SE or an SAP affiliate company. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# generate-debian-repo.sh +# +# Generates a Debian APT repository that conforms to the Debian Repository +# Format specification: https://wiki.debian.org/DebianRepository/Format +# +# Issue: https://github.com/SAP/SapMachine/issues/2216 +# +# The sapmachine apt repository at dist.sapmachine.io was non-compliant: +# 1. The suite was "./", which is not a real suite name and causes most +# tools (other than apt itself) to generate broken URLs. +# 2. The Release file was missing the required "Architecture:" field. +# +# This script generates a standards-compliant repository using: +# - Proper suite/codename naming (e.g. "stable", "testing") +# - Correct component naming (e.g. "main") +# - All required Release file fields including "Architecture:" +# - Packages index generated with dpkg-scanpackages +# - InRelease / Release.gpg files for GPG signature (if key is provided) +# +# Usage: +# generate-debian-repo.sh [OPTIONS] +# +# Options: +# -d, --deb-dir DIR Directory containing .deb packages (required) +# -o, --output-dir DIR Output directory for the repository (required) +# -s, --suite SUITE Suite name (default: "stable") +# -c, --component COMP Component name (default: "main") +# -a, --arch ARCH Architecture (default: auto-detected from packages) +# -k, --gpg-key KEY_ID GPG key ID for signing (optional) +# -O, --origin ORIGIN Origin field (default: "SapMachine") +# -L, --label LABEL Label field (default: "SapMachine") +# -D, --description DESC Description field +# -h, --help Show this help message + +set -euo pipefail + +# --------------------------------------------------------------------------- +# Defaults +# --------------------------------------------------------------------------- +DEB_DIR="" +OUTPUT_DIR="" +SUITE="stable" +COMPONENT="main" +ARCH="" +GPG_KEY_ID="" +ORIGIN="SapMachine" +LABEL="SapMachine" +DESCRIPTION="SAP SapMachine JDK Debian Repository" + +# --------------------------------------------------------------------------- +# Parse arguments +# --------------------------------------------------------------------------- +usage() { + grep '^#' "$0" | sed 's/^# \{0,1\}//' | grep -A1000 '^Usage:' | head -30 + exit 1 +} + +while [[ $# -gt 0 ]]; do + case "$1" in + -d|--deb-dir) DEB_DIR="$2"; shift 2 ;; + -o|--output-dir) OUTPUT_DIR="$2"; shift 2 ;; + -s|--suite) SUITE="$2"; shift 2 ;; + -c|--component) COMPONENT="$2"; shift 2 ;; + -a|--arch) ARCH="$2"; shift 2 ;; + -k|--gpg-key) GPG_KEY_ID="$2"; shift 2 ;; + -O|--origin) ORIGIN="$2"; shift 2 ;; + -L|--label) LABEL="$2"; shift 2 ;; + -D|--description) DESCRIPTION="$2"; shift 2 ;; + -h|--help) usage ;; + *) echo "Unknown option: $1" >&2; usage ;; + esac +done + +# --------------------------------------------------------------------------- +# Validate required arguments +# --------------------------------------------------------------------------- +if [[ -z "$DEB_DIR" ]]; then + echo "ERROR: --deb-dir is required." >&2 + exit 1 +fi +if [[ -z "$OUTPUT_DIR" ]]; then + echo "ERROR: --output-dir is required." >&2 + exit 1 +fi +if [[ ! -d "$DEB_DIR" ]]; then + echo "ERROR: deb-dir '$DEB_DIR' does not exist." >&2 + exit 1 +fi + +# Validate suite name: must not be "./" (which is non-compliant per the spec) +if [[ "$SUITE" == "./" ]] || [[ "$SUITE" == "." ]]; then + echo "ERROR: suite '$SUITE' is non-compliant. See https://wiki.debian.org/DebianRepository/Format" >&2 + echo " Use a proper suite name like 'stable', 'testing', 'sapmachine', etc." >&2 + exit 1 +fi + +# Validate suite name contains no special characters that would break paths +if [[ "$SUITE" =~ [[:space:]/] ]]; then + echo "ERROR: suite '$SUITE' must not contain spaces or slashes." >&2 + exit 1 +fi + +# --------------------------------------------------------------------------- +# Auto-detect architectures from .deb packages if not specified +# --------------------------------------------------------------------------- +detect_architectures() { + local deb_dir="$1" + local arches=() + for deb in "$deb_dir"/*.deb; do + [[ -f "$deb" ]] || continue + # dpkg-deb --info returns Architecture: field + local arch + arch=$(dpkg-deb --info "$deb" 2>/dev/null | grep '^[[:space:]]*Architecture:' | awk '{print $2}') + if [[ -n "$arch" ]]; then + arches+=("$arch") + fi + done + # Remove duplicates + printf '%s\n' "${arches[@]}" | sort -u | tr '\n' ' ' | sed 's/ $//' +} + +if [[ -z "$ARCH" ]]; then + if command -v dpkg-deb >/dev/null 2>&1; then + ARCH=$(detect_architectures "$DEB_DIR") + if [[ -z "$ARCH" ]]; then + ARCH="all" + echo "WARNING: No .deb packages found in '$DEB_DIR'; defaulting Architecture to 'all'" >&2 + fi + else + ARCH="amd64" + echo "WARNING: dpkg-deb not found; defaulting Architecture to 'amd64'" >&2 + fi +fi + +# --------------------------------------------------------------------------- +# Build the repository structure: +# $OUTPUT_DIR/dists/$SUITE/$COMPONENT/binary-$ARCH/ +# $OUTPUT_DIR/pool/$COMPONENT/ +# --------------------------------------------------------------------------- +echo "Generating Debian APT repository..." +echo " DEB directory : $DEB_DIR" +echo " Output dir : $OUTPUT_DIR" +echo " Suite : $SUITE" +echo " Component : $COMPONENT" +echo " Architecture : $ARCH" + +# Create directory structure +DISTS_DIR="$OUTPUT_DIR/dists/$SUITE" +POOL_DIR="$OUTPUT_DIR/pool/$COMPONENT" +mkdir -p "$POOL_DIR" + +# Copy .deb packages into the pool +shopt -s nullglob +deb_count=0 +for deb in "$DEB_DIR"/*.deb; do + cp -f "$deb" "$POOL_DIR/" + (( deb_count++ )) +done +echo " Copied $deb_count .deb package(s) to pool" + +# Generate per-architecture Packages index +for arch in $ARCH; do + BINARY_DIR="$DISTS_DIR/$COMPONENT/binary-$arch" + mkdir -p "$BINARY_DIR" + + if command -v dpkg-scanpackages >/dev/null 2>&1; then + # dpkg-scanpackages produces a Packages file relative to OUTPUT_DIR + pushd "$OUTPUT_DIR" > /dev/null + dpkg-scanpackages --arch "$arch" "pool/$COMPONENT" /dev/null 2>/dev/null \ + > "$BINARY_DIR/Packages" + gzip -9 -c "$BINARY_DIR/Packages" > "$BINARY_DIR/Packages.gz" + bzip2 -9 -c "$BINARY_DIR/Packages" > "$BINARY_DIR/Packages.bz2" + popd > /dev/null + echo " Generated Packages index for arch: $arch" + else + # Fallback when dpkg-scanpackages is not available (e.g. macOS CI without dpkg) + # Create empty but valid index files so the Release file can still be generated + echo "" > "$BINARY_DIR/Packages" + gzip -9 -c "$BINARY_DIR/Packages" > "$BINARY_DIR/Packages.gz" + bzip2 -9 -c "$BINARY_DIR/Packages" > "$BINARY_DIR/Packages.bz2" + echo " WARNING: dpkg-scanpackages not found; created empty Packages index for arch: $arch" >&2 + fi +done + +# --------------------------------------------------------------------------- +# Generate the Release file +# +# The Release file MUST include (per Debian spec): +# - Suite or Codename (we use Suite) +# - Components +# - Architecture ← this was MISSING in the old repo (issue #2216) +# - Date +# - MD5Sum / SHA1 / SHA256 checksums of all index files +# --------------------------------------------------------------------------- +RELEASE_FILE="$DISTS_DIR/Release" +DATE_RFC2822=$(date -Ru 2>/dev/null || date -u "+%a, %d %b %Y %H:%M:%S +0000") + +cat > "$RELEASE_FILE" < +generate_checksums() { + local hash_cmd="$1" + local section_header="$2" + local dists_dir="$3" + echo "$section_header" + find "$dists_dir" -mindepth 2 -type f \( -name 'Packages' -o -name 'Packages.gz' -o -name 'Packages.bz2' \) | \ + sort | while read -r file; do + local rel_path="${file#$dists_dir/}" + local size + size=$(wc -c < "$file") + local hash + hash=$($hash_cmd "$file" | awk '{print $1}') + printf " %s %d %s\n" "$hash" "$size" "$rel_path" + done +} + +generate_checksums "md5sum" "MD5Sum:" "$DISTS_DIR" >> "$RELEASE_FILE" +generate_checksums "sha1sum" "SHA1:" "$DISTS_DIR" >> "$RELEASE_FILE" +generate_checksums "sha256sum" "SHA256:" "$DISTS_DIR" >> "$RELEASE_FILE" + +echo " Generated Release file: $RELEASE_FILE" + +# --------------------------------------------------------------------------- +# Optionally sign the Release file with GPG +# --------------------------------------------------------------------------- +if [[ -n "$GPG_KEY_ID" ]]; then + if command -v gpg >/dev/null 2>&1; then + # Inline signature (InRelease) — preferred by modern apt + gpg --default-key "$GPG_KEY_ID" \ + --clearsign \ + --armor \ + --output "$DISTS_DIR/InRelease" \ + "$RELEASE_FILE" + echo " Generated signed InRelease file" + + # Detached signature (Release.gpg) — for compatibility + gpg --default-key "$GPG_KEY_ID" \ + --detach-sign \ + --armor \ + --output "$DISTS_DIR/Release.gpg" \ + "$RELEASE_FILE" + echo " Generated detached Release.gpg signature" + else + echo "WARNING: gpg not found; skipping signing." >&2 + fi +fi + +echo "Done. Repository at: $OUTPUT_DIR" +echo "" +echo "Example sources.list entry:" +echo " deb [arch=$ARCH] $SUITE $COMPONENT" diff --git a/make/scripts/test-generate-debian-repo.sh b/make/scripts/test-generate-debian-repo.sh new file mode 100755 index 000000000000..589ddd56e632 --- /dev/null +++ b/make/scripts/test-generate-debian-repo.sh @@ -0,0 +1,311 @@ +#!/usr/bin/env bash +# Copyright (c) 2024 SAP SE or an SAP affiliate company. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# test-generate-debian-repo.sh +# +# Unit tests for generate-debian-repo.sh +# +# Tests verify: +# 1. Argument validation (missing required args, invalid suite names) +# 2. Suite name validation — "./" must be rejected (issue #2216) +# 3. Release file structure — Architecture field is present (issue #2216) +# 4. Repository directory structure is correct (dists/$SUITE/$COMP/binary-$ARCH/) +# 5. All required Release file fields are present +# 6. Packages index files are created for each architecture +# +# Run: +# bash test-generate-debian-repo.sh +# or: +# ./test-generate-debian-repo.sh + +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SCRIPT="$SCRIPT_DIR/generate-debian-repo.sh" +TMPDIR_BASE="$(mktemp -d)" + +# --------------------------------------------------------------------------- +# Test helpers +# --------------------------------------------------------------------------- +PASS=0 +FAIL=0 + +pass() { echo " PASS: $1"; (( PASS++ )); } +fail() { echo " FAIL: $1"; (( FAIL++ )); } + +assert_eq() { + local desc="$1" expected="$2" actual="$3" + if [[ "$expected" == "$actual" ]]; then + pass "$desc" + else + fail "$desc (expected='$expected' actual='$actual')" + fi +} + +assert_contains() { + local desc="$1" needle="$2" haystack="$3" + if echo "$haystack" | grep -qF "$needle"; then + pass "$desc" + else + fail "$desc (expected to find '$needle' in output)" + fi +} + +assert_not_contains() { + local desc="$1" needle="$2" haystack="$3" + if ! echo "$haystack" | grep -qF "$needle"; then + pass "$desc" + else + fail "$desc (expected NOT to find '$needle' in output)" + fi +} + +assert_file_exists() { + local desc="$1" file="$2" + if [[ -f "$file" ]]; then + pass "$desc" + else + fail "$desc (file not found: $file)" + fi +} + +assert_file_not_exists() { + local desc="$1" file="$2" + if [[ ! -f "$file" ]]; then + pass "$desc" + else + fail "$desc (file should not exist: $file)" + fi +} + +assert_file_contains() { + local desc="$1" needle="$2" file="$3" + if [[ -f "$file" ]] && grep -qF "$needle" "$file"; then + pass "$desc" + else + fail "$desc (file '$file' does not contain '$needle')" + fi +} + +assert_exit_nonzero() { + local desc="$1" exit_code="$2" + if [[ "$exit_code" -ne 0 ]]; then + pass "$desc" + else + fail "$desc (expected non-zero exit, got 0)" + fi +} + +# --------------------------------------------------------------------------- +# Setup: create a minimal fake .deb for testing +# --------------------------------------------------------------------------- +create_fake_deb() { + local output_path="$1" + local arch="${2:-amd64}" + # Create a minimal DEBIAN/control structure and pack it + local tmp + tmp=$(mktemp -d) + mkdir -p "$tmp/DEBIAN" + cat > "$tmp/DEBIAN/control" < +Description: SapMachine JDK +EOF + if command -v dpkg-deb >/dev/null 2>&1; then + dpkg-deb --build "$tmp" "$output_path" >/dev/null 2>&1 + else + # Minimal .deb creation without dpkg-deb (for test environments) + # Create a valid .deb-like file using ar + touch "$output_path" + fi + rm -rf "$tmp" +} + +# --------------------------------------------------------------------------- +# Test suite +# --------------------------------------------------------------------------- + +echo "=== Unit Tests: generate-debian-repo.sh ===" +echo "" + +# --- Test 1: Missing required arguments --- +echo "--- Test group: Argument validation ---" + +output=$("$SCRIPT" 2>&1 || true) +exit_code=$("$SCRIPT" 2>&1; echo $?) || true +exit_code=$( "$SCRIPT" > /dev/null 2>&1; echo $? ) || true +# Use subshells to capture exit codes without triggering set -e +exit_code_no_args=0 +( "$SCRIPT" > /dev/null 2>&1 ) || exit_code_no_args=$? +assert_exit_nonzero "Missing --deb-dir fails with non-zero exit" "$exit_code_no_args" + +exit_code_no_output=0 +( "$SCRIPT" --deb-dir /tmp > /dev/null 2>&1 ) || exit_code_no_output=$? +assert_exit_nonzero "Missing --output-dir fails with non-zero exit" "$exit_code_no_output" + +# --- Test 2: Suite name validation (issue #2216) --- +echo "" +echo "--- Test group: Suite name validation (issue #2216) ---" + +DEBDIR="$TMPDIR_BASE/debs1" +OUTDIR="$TMPDIR_BASE/out1" +mkdir -p "$DEBDIR" + +# "./" should be rejected +exit_code=$( "$SCRIPT" --deb-dir "$DEBDIR" --output-dir "$OUTDIR" --suite "./" 2>/dev/null; echo $? ) || true +assert_exit_nonzero "Suite './' is rejected as non-compliant" "$exit_code" + +# "." should be rejected +exit_code=$( "$SCRIPT" --deb-dir "$DEBDIR" --output-dir "$OUTDIR" --suite "." 2>/dev/null; echo $? ) || true +assert_exit_nonzero "Suite '.' is rejected as non-compliant" "$exit_code" + +# Suite with "/" should be rejected +exit_code=$( "$SCRIPT" --deb-dir "$DEBDIR" --output-dir "$OUTDIR" --suite "foo/bar" 2>/dev/null; echo $? ) || true +assert_exit_nonzero "Suite 'foo/bar' (with slash) is rejected" "$exit_code" + +# Suite with space should be rejected +exit_code=$( "$SCRIPT" --deb-dir "$DEBDIR" --output-dir "$OUTDIR" --suite "foo bar" 2>/dev/null; echo $? ) || true +assert_exit_nonzero "Suite 'foo bar' (with space) is rejected" "$exit_code" + +# Valid suite names should be accepted (just validation, not full run) +error_output=$( "$SCRIPT" --deb-dir "$DEBDIR" --output-dir "$OUTDIR" --suite "stable" --arch "amd64" 2>&1 || true ) +assert_not_contains "Suite 'stable' is not rejected" \ + "ERROR: suite 'stable'" "$error_output" + +error_output=$( "$SCRIPT" --deb-dir "$DEBDIR" --output-dir "$OUTDIR" --suite "sapmachine" --arch "amd64" 2>&1 || true ) +assert_not_contains "Suite 'sapmachine' is not rejected" \ + "ERROR: suite 'sapmachine'" "$error_output" + +# --- Test 3: Generated Release file contains Architecture field (issue #2216) --- +echo "" +echo "--- Test group: Release file contains required fields ---" + +DEBDIR2="$TMPDIR_BASE/debs2" +OUTDIR2="$TMPDIR_BASE/out2" +mkdir -p "$DEBDIR2" +# Create a minimal fake .deb +create_fake_deb "$DEBDIR2/sapmachine-jdk-21_amd64.deb" "amd64" + +# Run the script +"$SCRIPT" \ + --deb-dir "$DEBDIR2" \ + --output-dir "$OUTDIR2" \ + --suite "sapmachine" \ + --component "main" \ + --arch "amd64" \ + --origin "SapMachine" \ + --label "SapMachine" \ + > /dev/null 2>&1 + +RELEASE_FILE="$OUTDIR2/dists/sapmachine/Release" + +assert_file_exists "Release file is generated" "$RELEASE_FILE" + +# Architecture field MUST be present (issue #2216 - it was missing) +assert_file_contains "Release file contains 'Architecture:' field (fix for issue #2216)" \ + "Architecture:" "$RELEASE_FILE" + +assert_file_contains "Release file contains 'Architecture: amd64'" \ + "Architecture: amd64" "$RELEASE_FILE" + +# Other required fields +assert_file_contains "Release file contains 'Suite:' field" \ + "Suite:" "$RELEASE_FILE" + +assert_file_contains "Release file has correct Suite value" \ + "Suite: sapmachine" "$RELEASE_FILE" + +assert_file_contains "Release file contains 'Components:' field" \ + "Components:" "$RELEASE_FILE" + +assert_file_contains "Release file contains 'Date:' field" \ + "Date:" "$RELEASE_FILE" + +assert_file_contains "Release file contains 'Origin:' field" \ + "Origin:" "$RELEASE_FILE" + +assert_file_contains "Release file contains 'Label:' field" \ + "Label:" "$RELEASE_FILE" + +# The suite must NOT be "./" (the broken value from issue #2216) +assert_not_contains "Release file Suite is NOT './'" \ + "Suite: ./" "$(cat "$RELEASE_FILE")" + +# --- Test 4: Repository directory structure --- +echo "" +echo "--- Test group: Repository directory structure ---" + +# pool/$COMPONENT is a directory not a file - check with -d +if [[ -d "$OUTDIR2/pool/main" ]]; then + pass "pool/main directory is created" +else + fail "pool/main directory is created (directory not found: $OUTDIR2/pool/main)" +fi + +# Packages index files +PACKAGES_FILE="$OUTDIR2/dists/sapmachine/main/binary-amd64/Packages" +assert_file_exists "Packages index is generated for amd64" "$PACKAGES_FILE" +assert_file_exists "Packages.gz is generated" "${PACKAGES_FILE}.gz" +assert_file_exists "Packages.bz2 is generated" "${PACKAGES_FILE}.bz2" + +# InRelease should NOT exist (no GPG key provided) +assert_file_not_exists "InRelease is NOT generated without GPG key" \ + "$OUTDIR2/dists/sapmachine/InRelease" + +# --- Test 5: Release file checksum sections --- +echo "" +echo "--- Test group: Release file checksum sections ---" + +assert_file_contains "Release file contains SHA256 section" \ + "SHA256:" "$RELEASE_FILE" + +assert_file_contains "Release file contains SHA1 section" \ + "SHA1:" "$RELEASE_FILE" + +assert_file_contains "Release file contains MD5Sum section" \ + "MD5Sum:" "$RELEASE_FILE" + +# --- Test 6: Custom suite and component --- +echo "" +echo "--- Test group: Custom suite and component names ---" + +DEBDIR3="$TMPDIR_BASE/debs3" +OUTDIR3="$TMPDIR_BASE/out3" +mkdir -p "$DEBDIR3" + +"$SCRIPT" \ + --deb-dir "$DEBDIR3" \ + --output-dir "$OUTDIR3" \ + --suite "lts" \ + --component "contrib" \ + --arch "arm64" \ + > /dev/null 2>&1 + +RELEASE_FILE3="$OUTDIR3/dists/lts/Release" +assert_file_exists "Release file created for custom suite 'lts'" "$RELEASE_FILE3" +assert_file_contains "Custom suite 'lts' is in Release file" "Suite: lts" "$RELEASE_FILE3" +assert_file_contains "Custom arch 'arm64' in Release file" "Architecture: arm64" "$RELEASE_FILE3" +assert_file_contains "Custom component 'contrib' in Release file" "Components: contrib" "$RELEASE_FILE3" + +# --- Cleanup --- +rm -rf "$TMPDIR_BASE" + +# --------------------------------------------------------------------------- +# Summary +# --------------------------------------------------------------------------- +echo "" +echo "=== Test Results ===" +echo " PASSED: $PASS" +echo " FAILED: $FAIL" +echo "" + +if [[ "$FAIL" -gt 0 ]]; then + echo "SOME TESTS FAILED" >&2 + exit 1 +else + echo "ALL TESTS PASSED" + exit 0 +fi