Have packages be built and test by CI for the release (only on master and tags)

.github/workflows/distribution.yml

@@ -0,0 +1,84 @@
+name: distribution
+permissions: read-all
+on:
+  push:
+    branches: [master]
+    tags:
+      - v[0-9]+.[0-9]+.[0-9]+** # Tag filters not as strict due to different regex system on Github Actions
+
+jobs:
+  build-dist:
+    runs-on: ubuntu-24.04
+    container:
+      image: quay.io/cortexproject/build-image:master-ee0b97cc37
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+      - name: Setup Git safe.directory
+        run: |
+          echo "this step is needed because when running in container, actions/checkout does not set safe.directory effectively."
+          echo "See https://github.com/actions/runner/issues/2033. We should use --system instead of --global"
+          git config --system --add safe.directory $GITHUB_WORKSPACE
+      - name: Sym Link Expected Path to Workspace
+        run: |
+          mkdir -p /go/src/github.com/cortexproject/cortex
+          ln -s $GITHUB_WORKSPACE/* /go/src/github.com/cortexproject/cortex
+      - name: Build Dist Binaries
+        run: |
+          touch build-image/.uptodate
+          make BUILD_IN_CONTAINER=false dist
+      - name: Upload Dist Binaries
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
+        with:
+          name: Dist Binaries
+          path: dist/
+
+  packages:
+    needs: build-dist
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+      - name: Download Dist Binaries
+        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
+        with:
+          name: Dist Binaries
+          path: dist
+      - name: Fix Permissions
+        run: chmod +x dist/cortex-* dist/query-tee-*
+      - name: Build Packages
+        run: |
+          touch dist/.uptodate
+          make TTY='' ARCHS=amd64 packages
+      - name: Upload Packages
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
+        with:
+          name: Packages
+          path: |
+            dist/*.deb
+            dist/*.rpm
+            dist/*-sha-256
+
+  test-packages:
+    needs: packages
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - runner: ubuntu-24.04
+            arch: amd64
+          - runner: ubuntu-24.04-arm
+            arch: arm64
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+      - name: Download Packages
+        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
+        with:
+          name: Packages
+          path: dist
+      - name: Build Test Images
+        run: make TTY='' ARCHS=${{ matrix.arch }} packaging/rpm/rockylinux-systemd/.uptodate packaging/deb/debian-systemd/.uptodate
+      - name: Test Packages
+        run: ./tools/packaging/test-packages quay.io/cortexproject/ $(cat VERSION) ${{ matrix.arch }}

Makefile

@@ -384,17 +384,19 @@ dist/$(UPTODATE)-packages: dist $(wildcard packaging/deb/**) $(wildcard packagin
 
 endif
 
-# Build both arm64 and amd64 images, so that we can test deb/rpm packages for both architectures.
-packaging/rpm/centos-systemd/$(UPTODATE): packaging/rpm/centos-systemd/Dockerfile
-	$(SUDO) docker build --platform linux/amd64 --build-arg=revision=$(GIT_REVISION) --build-arg=goproxyValue=$(GOPROXY_VALUE) -t $(IMAGE_PREFIX)$(shell basename $(@D)):amd64 $(@D)/
-	$(SUDO) docker build --platform linux/arm64 --build-arg=revision=$(GIT_REVISION) --build-arg=goproxyValue=$(GOPROXY_VALUE) -t $(IMAGE_PREFIX)$(shell basename $(@D)):arm64 $(@D)/
+# Build test images for the architectures specified by ARCHS.
+packaging/rpm/rockylinux-systemd/$(UPTODATE): packaging/rpm/rockylinux-systemd/Dockerfile
+	@for arch in $(ARCHS); do \
+		$(SUDO) docker build --platform linux/$$arch --build-arg=revision=$(GIT_REVISION) --build-arg=goproxyValue=$(GOPROXY_VALUE) -t $(IMAGE_PREFIX)$(shell basename $(@D)):$$arch $(@D)/ ; \
+	done
 	touch $@
 
 packaging/deb/debian-systemd/$(UPTODATE): packaging/deb/debian-systemd/Dockerfile
-	$(SUDO) docker build --platform linux/amd64 --build-arg=revision=$(GIT_REVISION) --build-arg=goproxyValue=$(GOPROXY_VALUE) -t $(IMAGE_PREFIX)$(shell basename $(@D)):amd64 $(@D)/
-	$(SUDO) docker build --platform linux/arm64 --build-arg=revision=$(GIT_REVISION) --build-arg=goproxyValue=$(GOPROXY_VALUE) -t $(IMAGE_PREFIX)$(shell basename $(@D)):arm64 $(@D)/
+	@for arch in $(ARCHS); do \
+		$(SUDO) docker build --platform linux/$$arch --build-arg=revision=$(GIT_REVISION) --build-arg=goproxyValue=$(GOPROXY_VALUE) -t $(IMAGE_PREFIX)$(shell basename $(@D)):$$arch $(@D)/ ; \
+	done
 	touch $@
 
 .PHONY: test-packages
-test-packages: packages packaging/rpm/centos-systemd/$(UPTODATE) packaging/deb/debian-systemd/$(UPTODATE)
-	./tools/packaging/test-packages $(IMAGE_PREFIX) $(VERSION)
+test-packages: packages packaging/rpm/rockylinux-systemd/$(UPTODATE) packaging/deb/debian-systemd/$(UPTODATE)
+	./tools/packaging/test-packages $(IMAGE_PREFIX) $(VERSION) $(ARCHS)

RELEASE.md

@@ -102,8 +102,8 @@ To publish a release candidate:
 1. Wait until CI pipeline succeeded (once a tag is created, the release process through GitHub actions will be triggered for this tag)
 1. Create a pre-release in GitHub
    - Write the release notes (including a copy-paste of the changelog)
-   - Build binaries with `make dist` and attach them to the release
-   - Build packages with `make packages`, test them with `make test-packages` and attach them to the release
+   - Download binaries from the `Dist Binaries` artifact in the [distribution](https://github.com/cortexproject/cortex/actions/workflows/distribution.yml) GitHub Actions workflow for the tag and attach them to the release
+   - Download packages from the `Packages` artifact in the [distribution](https://github.com/cortexproject/cortex/actions/workflows/distribution.yml) GitHub Actions workflow for the tag and attach them to the release
 1. [Sign the artifact and generate SBOM for the release](#sing-and-sbom)
 
 ### Publish a stable release
@@ -119,8 +119,8 @@ To publish a stable release:
 1. Wait until CI pipeline succeeded (once a tag is created, the release process through GitHub actions will be triggered for this tag)
 1. Create a release in GitHub
    - Write the release notes (including a copy-paste of the changelog)
-   - Build binaries with `make dist` and attach them to the release
-   - Build packages with `make packages`, test them with `make test-packages` and attach them to the release
+   - Download binaries from the `Dist Binaries` artifact in the [distribution](https://github.com/cortexproject/cortex/actions/workflows/distribution.yml) GitHub Actions workflow for the tag and attach them to the release
+   - Download packages from the `Packages` artifact in the [distribution](https://github.com/cortexproject/cortex/actions/workflows/distribution.yml) GitHub Actions workflow for the tag and attach them to the release
 1. [Sign the artifact and generate SBOM for the release](#sing-and-sbom)
 1. Merge the release branch `release-x.y` to `master`
    - Create `merge-release-X.Y-to-master` branch **from `release-X.Y` branch** locally

packaging/deb/debian-systemd/Dockerfile

@@ -1,4 +1,4 @@
-FROM debian:10
+FROM debian:12
 ENV container docker
 ENV LC_ALL C
 ENV DEBIAN_FRONTEND noninteractive

packaging/fpm/Dockerfile

@@ -3,7 +3,6 @@ FROM alpine:3.23
 RUN apk add --no-cache \
         ruby \
         ruby-dev \
-        ruby-etc \
         gcc \
         git \
         libc-dev \

packaging/rpm/rockylinux-systemd/Dockerfile

@@ -0,0 +1,14 @@
+FROM rockylinux:9
+ENV container docker
+RUN dnf -y install systemd && dnf clean all && \
+        (cd /usr/lib/systemd/system/sysinit.target.wants/; for i in *; do [ "$i" = \
+        systemd-tmpfiles-setup.service ] || rm -f "$i"; done); \
+        rm -f /usr/lib/systemd/system/multi-user.target.wants/*; \
+        rm -f /etc/systemd/system/*.wants/*; \
+        rm -f /usr/lib/systemd/system/local-fs.target.wants/*; \
+        rm -f /usr/lib/systemd/system/sockets.target.wants/*udev*; \
+        rm -f /usr/lib/systemd/system/sockets.target.wants/*initctl*; \
+        rm -f /usr/lib/systemd/system/basic.target.wants/*;
+
+VOLUME [ "/sys/fs/cgroup"]
+CMD ["/usr/sbin/init"]

tools/packaging/test-packages

@@ -4,6 +4,8 @@ set -euf -o pipefail
 
 readonly IMAGE_PREFIX=$1
 readonly VERSION=$2
+shift 2
+readonly ARCHS=("$@")
 readonly DISABLE_CLEANUP=${DISABLE_CLEANUP:-0}
 
 declare -a CONTAINERS=()
@@ -46,18 +48,34 @@ function test_with_systemd() {
 
     echo "Testing $install_command on $image ($platform)"
 
-    container=$(docker run --platform="${platform}" --tmpfs /run --tmpfs /run/lock -v /sys/fs/cgroup:/sys/fs/cgroup:ro -itd -v "$(pwd)"/dist:/opt/cortex -p 9009 "${image}")
+    container=$(docker run --platform="${platform}" --privileged --cgroupns=host --tmpfs /run --tmpfs /run/lock -v /sys/fs/cgroup:/sys/fs/cgroup:rw -td -v "$(pwd)"/dist:/opt/cortex -v "$(pwd)"/docs/configuration:/opt/config:ro -p 9009 "${image}")
     CONTAINERS+=("${container}")
 
-    port=$(docker inspect --format='{{(index (index .NetworkSettings.Ports "9009/tcp") 0).HostPort}}' "${container}")
+    # Wait briefly for systemd to initialize and verify container is running
+    sleep 2
+    if ! docker inspect --format='{{.State.Running}}' "${container}" | grep -q true; then
+        echo "Container failed to start:"
+        docker logs "${container}" || true
+        error "Container ${container} is not running"
+    fi
 
-    docker exec -it "${container}" /bin/bash -c "${install_command}; systemctl start cortex.service; systemctl enable cortex.service"
+    port=$(docker inspect --format='{{(index (index .NetworkSettings.Ports "9009/tcp") 0).HostPort}}' "${container}")
 
-    ready 10 1 "${port}" || error "Testing image: ${image} with command: '${install_command}' failed"
+    docker exec "${container}" /bin/bash -c "${install_command}; mkdir -p /tmp/cortex/tsdb /tmp/cortex/tsdb-sync /tmp/cortex/compactor /tmp/cortex/rules /tmp/cortex/alerts /var/lib/cortex/data/tsdb && chown -R cortex:cortex /tmp/cortex /var/lib/cortex; cp /opt/config/single-process-config-blocks-local.yaml /etc/cortex/single-process-config.yaml && chown root:cortex /etc/cortex/single-process-config.yaml; systemctl start cortex.service; systemctl enable cortex.service"
+
+    ready 30 2 "${port}" || {
+        echo "--- curl localhost:${port}/ready ---"
+        curl -s localhost:"${port}"/ready || true
+        echo ""
+        echo "--- systemctl status cortex.service ---"
+        docker exec "${container}" systemctl status cortex.service || true
+        echo "--- journalctl -u cortex.service ---"
+        docker exec "${container}" journalctl -u cortex.service --no-pager -n 50 || true
+        error "Testing image: ${image} with command: '${install_command}' failed"
+    }
 }
 
-test_with_systemd "${IMAGE_PREFIX}"debian-systemd:amd64 linux/amd64 "dpkg -i /opt/cortex/cortex-${VERSION}_amd64.deb"
-test_with_systemd "${IMAGE_PREFIX}"debian-systemd:arm64 linux/arm64 "dpkg -i /opt/cortex/cortex-${VERSION}_arm64.deb"
-
-test_with_systemd "${IMAGE_PREFIX}"centos-systemd:amd64 linux/amd64 "rpm -i /opt/cortex/cortex-${VERSION}_amd64.rpm"
-test_with_systemd "${IMAGE_PREFIX}"centos-systemd:arm64 linux/arm64 "rpm -i /opt/cortex/cortex-${VERSION}_arm64.rpm"
+for arch in "${ARCHS[@]}"; do
+    test_with_systemd "${IMAGE_PREFIX}"debian-systemd:"${arch}" linux/"${arch}" "dpkg -i /opt/cortex/cortex-${VERSION}_${arch}.deb"
+    test_with_systemd "${IMAGE_PREFIX}"rockylinux-systemd:"${arch}" linux/"${arch}" "rpm -i /opt/cortex/cortex-${VERSION}_${arch}.rpm"
+done