From 855e8e936dfa224aeaad3b6a36a190b9b273a4da Mon Sep 17 00:00:00 2001 From: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> Date: Fri, 13 Mar 2026 13:04:52 -0700 Subject: [PATCH 01/17] Have packages be built by CI Signed-off-by: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> --- .github/workflows/test-build-deploy.yml | 34 ++++++++++++++++++++++++- packaging/fpm/Dockerfile | 1 - 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-build-deploy.yml b/.github/workflows/test-build-deploy.yml index a0541ae3bd5..d74fe7e87bf 100644 --- a/.github/workflows/test-build-deploy.yml +++ b/.github/workflows/test-build-deploy.yml @@ -313,8 +313,40 @@ jobs: echo "Running configs integration tests on ${{ matrix.arch }}" make BUILD_IMAGE=quay.io/cortexproject/build-image:master-ee0b97cc37 TTY='' configs-integration-test + packages: + needs: lint + runs-on: ubuntu-24.04 + container: + image: quay.io/cortexproject/build-image:master-fe84258322 + steps: + - name: Checkout Repo + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - name: Setup Git safe.directory + run: | + echo "this step is needed because when running in container, actions/checkout does not set safe.directory effectively." + echo "See https://github.com/actions/runner/issues/2033. We should use --system instead of --global" + git config --system --add safe.directory $GITHUB_WORKSPACE + - name: Sym Link Expected Path to Workspace + run: | + mkdir -p /go/src/github.com/cortexproject/cortex + ln -s $GITHUB_WORKSPACE/* /go/src/github.com/cortexproject/cortex + - name: Build Binaries + run: | + touch build-image/.uptodate + make BUILD_IN_CONTAINER=false dist + - name: Build Packages + run: make BUILD_IN_CONTAINER=false PACKAGE_IN_CONTAINER=false packages + - name: Upload Packages + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: Packages + path: | + dist/*.deb + dist/*.rpm + dist/*-sha-256 + deploy: - needs: [build, test, lint, integration, integration-configs-db] + needs: [build, test, lint, integration, integration-configs-db, packages] if: (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')) && github.repository == 'cortexproject/cortex' runs-on: ubuntu-24.04 container: diff --git a/packaging/fpm/Dockerfile b/packaging/fpm/Dockerfile index 2b32eea23d4..273d2d3e601 100644 --- a/packaging/fpm/Dockerfile +++ b/packaging/fpm/Dockerfile @@ -3,7 +3,6 @@ FROM alpine:3.23 RUN apk add --no-cache \ ruby \ ruby-dev \ - ruby-etc \ gcc \ git \ libc-dev \ From 9e7594bd6557e016f5935bc782ca03526ab076ca Mon Sep 17 00:00:00 2001 From: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> Date: Mon, 20 Apr 2026 11:17:39 -0700 Subject: [PATCH 02/17] Fix package testing for modern base images and improve debugging - Upgrade deb test image from debian:10 to debian:12 - Migrate rpm test image from centos:8 (EOL) to rockylinux:9 - Fix systemd unit paths for Rocky Linux (/lib -> /usr/lib) - Add proper Cortex config and data directories in test containers - Increase readiness timeout and add diagnostic output on failure - Run containers in privileged mode for systemd compatibility Signed-off-by: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> --- packaging/deb/debian-systemd/Dockerfile | 2 +- packaging/rpm/centos-systemd/Dockerfile | 18 +++++++++--------- tools/packaging/test-packages | 17 +++++++++++++---- 3 files changed, 23 insertions(+), 14 deletions(-) diff --git a/packaging/deb/debian-systemd/Dockerfile b/packaging/deb/debian-systemd/Dockerfile index 3946fd8a8ff..b3b28116cad 100644 --- a/packaging/deb/debian-systemd/Dockerfile +++ b/packaging/deb/debian-systemd/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:10 +FROM debian:12 ENV container docker ENV LC_ALL C ENV DEBIAN_FRONTEND noninteractive diff --git a/packaging/rpm/centos-systemd/Dockerfile b/packaging/rpm/centos-systemd/Dockerfile index 6996b323cd8..3d690c4f2a1 100644 --- a/packaging/rpm/centos-systemd/Dockerfile +++ b/packaging/rpm/centos-systemd/Dockerfile @@ -1,14 +1,14 @@ -FROM centos:8 +FROM rockylinux:9 ENV container docker -RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i == \ - systemd-tmpfiles-setup.service ] || rm -f $i; done); \ - rm -f /lib/systemd/system/multi-user.target.wants/*; \ +RUN dnf -y install systemd && dnf clean all && \ + (cd /usr/lib/systemd/system/sysinit.target.wants/; for i in *; do [ "$i" = \ + systemd-tmpfiles-setup.service ] || rm -f "$i"; done); \ + rm -f /usr/lib/systemd/system/multi-user.target.wants/*; \ rm -f /etc/systemd/system/*.wants/*; \ - rm -f /lib/systemd/system/local-fs.target.wants/*; \ - rm -f /lib/systemd/system/sockets.target.wants/*udev*; \ - rm -f /lib/systemd/system/sockets.target.wants/*initctl*; \ - rm -f /lib/systemd/system/basic.target.wants/*; \ - rm -f /lib/systemd/system/anaconda.target.wants/*; + rm -f /usr/lib/systemd/system/local-fs.target.wants/*; \ + rm -f /usr/lib/systemd/system/sockets.target.wants/*udev*; \ + rm -f /usr/lib/systemd/system/sockets.target.wants/*initctl*; \ + rm -f /usr/lib/systemd/system/basic.target.wants/*; VOLUME [ "/sys/fs/cgroup"] CMD ["/usr/sbin/init"] diff --git a/tools/packaging/test-packages b/tools/packaging/test-packages index 7a81a7345d1..d2e60b56c68 100755 --- a/tools/packaging/test-packages +++ b/tools/packaging/test-packages @@ -46,14 +46,23 @@ function test_with_systemd() { echo "Testing $install_command on $image ($platform)" - container=$(docker run --platform="${platform}" --tmpfs /run --tmpfs /run/lock -v /sys/fs/cgroup:/sys/fs/cgroup:ro -itd -v "$(pwd)"/dist:/opt/cortex -p 9009 "${image}") + container=$(docker run --platform="${platform}" --privileged --tmpfs /run --tmpfs /run/lock -v /sys/fs/cgroup:/sys/fs/cgroup -td -v "$(pwd)"/dist:/opt/cortex -v "$(pwd)"/docs/configuration:/opt/config:ro -p 9009 "${image}") CONTAINERS+=("${container}") port=$(docker inspect --format='{{(index (index .NetworkSettings.Ports "9009/tcp") 0).HostPort}}' "${container}") - docker exec -it "${container}" /bin/bash -c "${install_command}; systemctl start cortex.service; systemctl enable cortex.service" - - ready 10 1 "${port}" || error "Testing image: ${image} with command: '${install_command}' failed" + docker exec "${container}" /bin/bash -c "${install_command}; mkdir -p /tmp/cortex/tsdb /tmp/cortex/tsdb-sync /tmp/cortex/compactor /tmp/cortex/rules /tmp/cortex/alerts /var/lib/cortex/data/tsdb && chown -R cortex:cortex /tmp/cortex /var/lib/cortex; cp /opt/config/single-process-config-blocks-local.yaml /etc/cortex/single-process-config.yaml && chown root:cortex /etc/cortex/single-process-config.yaml; systemctl start cortex.service; systemctl enable cortex.service" + + ready 30 2 "${port}" || { + echo "--- curl localhost:${port}/ready ---" + curl -s localhost:"${port}"/ready || true + echo "" + echo "--- systemctl status cortex.service ---" + docker exec "${container}" systemctl status cortex.service || true + echo "--- journalctl -u cortex.service ---" + docker exec "${container}" journalctl -u cortex.service --no-pager -n 50 || true + error "Testing image: ${image} with command: '${install_command}' failed" + } } test_with_systemd "${IMAGE_PREFIX}"debian-systemd:amd64 linux/amd64 "dpkg -i /opt/cortex/cortex-${VERSION}_amd64.deb" From 5c306f6c720f05df5b501410b4545fd1254e8177 Mon Sep 17 00:00:00 2001 From: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> Date: Mon, 20 Apr 2026 11:24:04 -0700 Subject: [PATCH 03/17] Rename centos-systemd to rockylinux-systemd The base image was migrated from centos:8 to rockylinux:9 in the previous commit. Rename the directory and update all references in the Makefile and test-packages script to match. Signed-off-by: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> --- Makefile | 4 ++-- .../rpm/{centos-systemd => rockylinux-systemd}/Dockerfile | 0 tools/packaging/test-packages | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) rename packaging/rpm/{centos-systemd => rockylinux-systemd}/Dockerfile (100%) diff --git a/Makefile b/Makefile index 9c85ae5ddc1..f9521ff3fb0 100644 --- a/Makefile +++ b/Makefile @@ -385,7 +385,7 @@ dist/$(UPTODATE)-packages: dist $(wildcard packaging/deb/**) $(wildcard packagin endif # Build both arm64 and amd64 images, so that we can test deb/rpm packages for both architectures. -packaging/rpm/centos-systemd/$(UPTODATE): packaging/rpm/centos-systemd/Dockerfile +packaging/rpm/rockylinux-systemd/$(UPTODATE): packaging/rpm/rockylinux-systemd/Dockerfile $(SUDO) docker build --platform linux/amd64 --build-arg=revision=$(GIT_REVISION) --build-arg=goproxyValue=$(GOPROXY_VALUE) -t $(IMAGE_PREFIX)$(shell basename $(@D)):amd64 $(@D)/ $(SUDO) docker build --platform linux/arm64 --build-arg=revision=$(GIT_REVISION) --build-arg=goproxyValue=$(GOPROXY_VALUE) -t $(IMAGE_PREFIX)$(shell basename $(@D)):arm64 $(@D)/ touch $@ @@ -396,5 +396,5 @@ packaging/deb/debian-systemd/$(UPTODATE): packaging/deb/debian-systemd/Dockerfil touch $@ .PHONY: test-packages -test-packages: packages packaging/rpm/centos-systemd/$(UPTODATE) packaging/deb/debian-systemd/$(UPTODATE) +test-packages: packages packaging/rpm/rockylinux-systemd/$(UPTODATE) packaging/deb/debian-systemd/$(UPTODATE) ./tools/packaging/test-packages $(IMAGE_PREFIX) $(VERSION) diff --git a/packaging/rpm/centos-systemd/Dockerfile b/packaging/rpm/rockylinux-systemd/Dockerfile similarity index 100% rename from packaging/rpm/centos-systemd/Dockerfile rename to packaging/rpm/rockylinux-systemd/Dockerfile diff --git a/tools/packaging/test-packages b/tools/packaging/test-packages index d2e60b56c68..1b9c1e02cce 100755 --- a/tools/packaging/test-packages +++ b/tools/packaging/test-packages @@ -68,5 +68,5 @@ function test_with_systemd() { test_with_systemd "${IMAGE_PREFIX}"debian-systemd:amd64 linux/amd64 "dpkg -i /opt/cortex/cortex-${VERSION}_amd64.deb" test_with_systemd "${IMAGE_PREFIX}"debian-systemd:arm64 linux/arm64 "dpkg -i /opt/cortex/cortex-${VERSION}_arm64.deb" -test_with_systemd "${IMAGE_PREFIX}"centos-systemd:amd64 linux/amd64 "rpm -i /opt/cortex/cortex-${VERSION}_amd64.rpm" -test_with_systemd "${IMAGE_PREFIX}"centos-systemd:arm64 linux/arm64 "rpm -i /opt/cortex/cortex-${VERSION}_arm64.rpm" +test_with_systemd "${IMAGE_PREFIX}"rockylinux-systemd:amd64 linux/amd64 "rpm -i /opt/cortex/cortex-${VERSION}_amd64.rpm" +test_with_systemd "${IMAGE_PREFIX}"rockylinux-systemd:arm64 linux/arm64 "rpm -i /opt/cortex/cortex-${VERSION}_arm64.rpm" From e2e30b64535770f717a76ebe23cfa53bf1894532 Mon Sep 17 00:00:00 2001 From: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> Date: Mon, 20 Apr 2026 11:34:58 -0700 Subject: [PATCH 04/17] create test-packages step Signed-off-by: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> --- .github/workflows/test-build-deploy.yml | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-build-deploy.yml b/.github/workflows/test-build-deploy.yml index d74fe7e87bf..3c47906b8ed 100644 --- a/.github/workflows/test-build-deploy.yml +++ b/.github/workflows/test-build-deploy.yml @@ -345,8 +345,26 @@ jobs: dist/*.rpm dist/*-sha-256 + test-packages: + needs: packages + runs-on: ubuntu-24.04 + steps: + - name: Checkout Repo + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - name: Set up QEMU + uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0 + - name: Download Packages + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: Packages + path: dist + - name: Test Packages + run: | + touch dist/.uptodate-packages + make PACKAGE_IN_CONTAINER=false test-packages + deploy: - needs: [build, test, lint, integration, integration-configs-db, packages] + needs: [build, test, lint, integration, integration-configs-db, packages, test-packages] if: (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')) && github.repository == 'cortexproject/cortex' runs-on: ubuntu-24.04 container: From 6f396b53341cf716b31630106a9d87fe640d0096 Mon Sep 17 00:00:00 2001 From: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> Date: Mon, 20 Apr 2026 11:41:35 -0700 Subject: [PATCH 05/17] Correctly build packages Signed-off-by: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> --- .github/workflows/test-build-deploy.yml | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/.github/workflows/test-build-deploy.yml b/.github/workflows/test-build-deploy.yml index 3c47906b8ed..d2a838fe771 100644 --- a/.github/workflows/test-build-deploy.yml +++ b/.github/workflows/test-build-deploy.yml @@ -316,26 +316,13 @@ jobs: packages: needs: lint runs-on: ubuntu-24.04 - container: - image: quay.io/cortexproject/build-image:master-fe84258322 steps: - name: Checkout Repo uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - - name: Setup Git safe.directory - run: | - echo "this step is needed because when running in container, actions/checkout does not set safe.directory effectively." - echo "See https://github.com/actions/runner/issues/2033. We should use --system instead of --global" - git config --system --add safe.directory $GITHUB_WORKSPACE - - name: Sym Link Expected Path to Workspace - run: | - mkdir -p /go/src/github.com/cortexproject/cortex - ln -s $GITHUB_WORKSPACE/* /go/src/github.com/cortexproject/cortex - name: Build Binaries - run: | - touch build-image/.uptodate - make BUILD_IN_CONTAINER=false dist + run: make dist - name: Build Packages - run: make BUILD_IN_CONTAINER=false PACKAGE_IN_CONTAINER=false packages + run: make packages - name: Upload Packages uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: From 140fe74deebe5ab35c215117d6daf2ccf1fe487a Mon Sep 17 00:00:00 2001 From: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> Date: Mon, 20 Apr 2026 12:13:21 -0700 Subject: [PATCH 06/17] Fix build issues Signed-off-by: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> --- .github/workflows/test-build-deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-build-deploy.yml b/.github/workflows/test-build-deploy.yml index d2a838fe771..d4af7e9175a 100644 --- a/.github/workflows/test-build-deploy.yml +++ b/.github/workflows/test-build-deploy.yml @@ -322,7 +322,7 @@ jobs: - name: Build Binaries run: make dist - name: Build Packages - run: make packages + run: make TTY='' ARCHS=amd64 packages - name: Upload Packages uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: From a142064f89c59b7dfd7d812e0a271bb4909cfeb6 Mon Sep 17 00:00:00 2001 From: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> Date: Mon, 20 Apr 2026 13:52:12 -0700 Subject: [PATCH 07/17] Build the binaries once and re-use them Signed-off-by: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> --- .github/workflows/test-build-deploy.yml | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-build-deploy.yml b/.github/workflows/test-build-deploy.yml index d4af7e9175a..194453267b2 100644 --- a/.github/workflows/test-build-deploy.yml +++ b/.github/workflows/test-build-deploy.yml @@ -155,6 +155,13 @@ jobs: with: name: Docker Images path: ./images.tar + - name: Build Dist Binaries + run: make BUILD_IN_CONTAINER=false dist + - name: Upload Dist Binaries + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: Dist Binaries + path: dist/ integration: needs: [build, lint] @@ -314,13 +321,16 @@ jobs: make BUILD_IMAGE=quay.io/cortexproject/build-image:master-ee0b97cc37 TTY='' configs-integration-test packages: - needs: lint + needs: build runs-on: ubuntu-24.04 steps: - name: Checkout Repo uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - - name: Build Binaries - run: make dist + - name: Download Dist Binaries + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: Dist Binaries + path: dist - name: Build Packages run: make TTY='' ARCHS=amd64 packages - name: Upload Packages @@ -348,7 +358,7 @@ jobs: - name: Test Packages run: | touch dist/.uptodate-packages - make PACKAGE_IN_CONTAINER=false test-packages + make TTY='' test-packages deploy: needs: [build, test, lint, integration, integration-configs-db, packages, test-packages] From 01d4463c84ad48496ec56e331995212d80b0c55e Mon Sep 17 00:00:00 2001 From: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> Date: Mon, 20 Apr 2026 14:17:56 -0700 Subject: [PATCH 08/17] Prevent packages being rebuilt Signed-off-by: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> --- .github/workflows/test-build-deploy.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-build-deploy.yml b/.github/workflows/test-build-deploy.yml index 194453267b2..f28ef4c12ff 100644 --- a/.github/workflows/test-build-deploy.yml +++ b/.github/workflows/test-build-deploy.yml @@ -332,7 +332,9 @@ jobs: name: Dist Binaries path: dist - name: Build Packages - run: make TTY='' ARCHS=amd64 packages + run: | + touch dist/.uptodate + make TTY='' ARCHS=amd64 packages - name: Upload Packages uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: @@ -357,6 +359,7 @@ jobs: path: dist - name: Test Packages run: | + touch dist/.uptodate touch dist/.uptodate-packages make TTY='' test-packages From 58ec1bfed10d5870d5bf5efc02a94e4036e6922f Mon Sep 17 00:00:00 2001 From: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> Date: Mon, 20 Apr 2026 14:37:05 -0700 Subject: [PATCH 09/17] Split to build-dist the building step Signed-off-by: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> --- .github/workflows/test-build-deploy.yml | 26 ++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-build-deploy.yml b/.github/workflows/test-build-deploy.yml index f28ef4c12ff..a938c540af8 100644 --- a/.github/workflows/test-build-deploy.yml +++ b/.github/workflows/test-build-deploy.yml @@ -155,8 +155,28 @@ jobs: with: name: Docker Images path: ./images.tar + + build-dist: + needs: lint + runs-on: ubuntu-24.04 + container: + image: quay.io/cortexproject/build-image:master-ee0b97cc37 + steps: + - name: Checkout Repo + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - name: Setup Git safe.directory + run: | + echo "this step is needed because when running in container, actions/checkout does not set safe.directory effectively." + echo "See https://github.com/actions/runner/issues/2033. We should use --system instead of --global" + git config --system --add safe.directory $GITHUB_WORKSPACE + - name: Sym Link Expected Path to Workspace + run: | + mkdir -p /go/src/github.com/cortexproject/cortex + ln -s $GITHUB_WORKSPACE/* /go/src/github.com/cortexproject/cortex - name: Build Dist Binaries - run: make BUILD_IN_CONTAINER=false dist + run: | + touch build-image/.uptodate + make BUILD_IN_CONTAINER=false dist - name: Upload Dist Binaries uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: @@ -321,7 +341,7 @@ jobs: make BUILD_IMAGE=quay.io/cortexproject/build-image:master-ee0b97cc37 TTY='' configs-integration-test packages: - needs: build + needs: build-dist runs-on: ubuntu-24.04 steps: - name: Checkout Repo @@ -361,7 +381,7 @@ jobs: run: | touch dist/.uptodate touch dist/.uptodate-packages - make TTY='' test-packages + make TTY='' PACKAGE_IN_CONTAINER=false test-packages deploy: needs: [build, test, lint, integration, integration-configs-db, packages, test-packages] From e6e407303efa08fecf48ac03d05bfe9bee7ec63d Mon Sep 17 00:00:00 2001 From: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> Date: Mon, 20 Apr 2026 15:02:27 -0700 Subject: [PATCH 10/17] let's just run the test Signed-off-by: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> --- .github/workflows/test-build-deploy.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test-build-deploy.yml b/.github/workflows/test-build-deploy.yml index a938c540af8..8e33f258dbd 100644 --- a/.github/workflows/test-build-deploy.yml +++ b/.github/workflows/test-build-deploy.yml @@ -157,7 +157,6 @@ jobs: path: ./images.tar build-dist: - needs: lint runs-on: ubuntu-24.04 container: image: quay.io/cortexproject/build-image:master-ee0b97cc37 @@ -377,11 +376,10 @@ jobs: with: name: Packages path: dist + - name: Build Test Images + run: make TTY='' packaging/rpm/rockylinux-systemd/.uptodate packaging/deb/debian-systemd/.uptodate - name: Test Packages - run: | - touch dist/.uptodate - touch dist/.uptodate-packages - make TTY='' PACKAGE_IN_CONTAINER=false test-packages + run: ./tools/packaging/test-packages quay.io/cortexproject/ $(cat VERSION) deploy: needs: [build, test, lint, integration, integration-configs-db, packages, test-packages] From 075b400602b2749802aabca8fb55d197a52f1e70 Mon Sep 17 00:00:00 2001 From: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> Date: Mon, 20 Apr 2026 15:23:00 -0700 Subject: [PATCH 11/17] avoid using qemu for this Signed-off-by: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> --- .github/workflows/test-build-deploy.yml | 18 ++++++++++++------ Makefile | 2 +- tools/packaging/test-packages | 11 ++++++----- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/.github/workflows/test-build-deploy.yml b/.github/workflows/test-build-deploy.yml index 8e33f258dbd..4d00e14af64 100644 --- a/.github/workflows/test-build-deploy.yml +++ b/.github/workflows/test-build-deploy.yml @@ -365,24 +365,30 @@ jobs: test-packages: needs: packages - runs-on: ubuntu-24.04 + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + include: + - runner: ubuntu-24.04 + arch: amd64 + - runner: ubuntu-24.04-arm + arch: arm64 steps: - name: Checkout Repo uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - - name: Set up QEMU - uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0 - name: Download Packages uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: Packages path: dist - name: Build Test Images - run: make TTY='' packaging/rpm/rockylinux-systemd/.uptodate packaging/deb/debian-systemd/.uptodate + run: make TTY='' ARCHS=${{ matrix.arch }} packaging/rpm/rockylinux-systemd/.uptodate packaging/deb/debian-systemd/.uptodate - name: Test Packages - run: ./tools/packaging/test-packages quay.io/cortexproject/ $(cat VERSION) + run: ./tools/packaging/test-packages quay.io/cortexproject/ $(cat VERSION) ${{ matrix.arch }} deploy: - needs: [build, test, lint, integration, integration-configs-db, packages, test-packages] + needs: [build, test, lint, integration, integration-configs-db] if: (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')) && github.repository == 'cortexproject/cortex' runs-on: ubuntu-24.04 container: diff --git a/Makefile b/Makefile index f9521ff3fb0..7b99cf89028 100644 --- a/Makefile +++ b/Makefile @@ -397,4 +397,4 @@ packaging/deb/debian-systemd/$(UPTODATE): packaging/deb/debian-systemd/Dockerfil .PHONY: test-packages test-packages: packages packaging/rpm/rockylinux-systemd/$(UPTODATE) packaging/deb/debian-systemd/$(UPTODATE) - ./tools/packaging/test-packages $(IMAGE_PREFIX) $(VERSION) + ./tools/packaging/test-packages $(IMAGE_PREFIX) $(VERSION) $(ARCHS) diff --git a/tools/packaging/test-packages b/tools/packaging/test-packages index 1b9c1e02cce..bc1392499f3 100755 --- a/tools/packaging/test-packages +++ b/tools/packaging/test-packages @@ -4,6 +4,8 @@ set -euf -o pipefail readonly IMAGE_PREFIX=$1 readonly VERSION=$2 +shift 2 +readonly ARCHS=("$@") readonly DISABLE_CLEANUP=${DISABLE_CLEANUP:-0} declare -a CONTAINERS=() @@ -65,8 +67,7 @@ function test_with_systemd() { } } -test_with_systemd "${IMAGE_PREFIX}"debian-systemd:amd64 linux/amd64 "dpkg -i /opt/cortex/cortex-${VERSION}_amd64.deb" -test_with_systemd "${IMAGE_PREFIX}"debian-systemd:arm64 linux/arm64 "dpkg -i /opt/cortex/cortex-${VERSION}_arm64.deb" - -test_with_systemd "${IMAGE_PREFIX}"rockylinux-systemd:amd64 linux/amd64 "rpm -i /opt/cortex/cortex-${VERSION}_amd64.rpm" -test_with_systemd "${IMAGE_PREFIX}"rockylinux-systemd:arm64 linux/arm64 "rpm -i /opt/cortex/cortex-${VERSION}_arm64.rpm" +for arch in "${ARCHS[@]}"; do + test_with_systemd "${IMAGE_PREFIX}"debian-systemd:"${arch}" linux/"${arch}" "dpkg -i /opt/cortex/cortex-${VERSION}_${arch}.deb" + test_with_systemd "${IMAGE_PREFIX}"rockylinux-systemd:"${arch}" linux/"${arch}" "rpm -i /opt/cortex/cortex-${VERSION}_${arch}.rpm" +done From faee918244f91adb2fc7a3e65d57a4e4c31a91cb Mon Sep 17 00:00:00 2001 From: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> Date: Mon, 20 Apr 2026 15:38:24 -0700 Subject: [PATCH 12/17] Now each runner only builds the test image for its own architecture. Signed-off-by: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> --- Makefile | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 7b99cf89028..fc684adc4f8 100644 --- a/Makefile +++ b/Makefile @@ -384,15 +384,17 @@ dist/$(UPTODATE)-packages: dist $(wildcard packaging/deb/**) $(wildcard packagin endif -# Build both arm64 and amd64 images, so that we can test deb/rpm packages for both architectures. +# Build test images for the architectures specified by ARCHS. packaging/rpm/rockylinux-systemd/$(UPTODATE): packaging/rpm/rockylinux-systemd/Dockerfile - $(SUDO) docker build --platform linux/amd64 --build-arg=revision=$(GIT_REVISION) --build-arg=goproxyValue=$(GOPROXY_VALUE) -t $(IMAGE_PREFIX)$(shell basename $(@D)):amd64 $(@D)/ - $(SUDO) docker build --platform linux/arm64 --build-arg=revision=$(GIT_REVISION) --build-arg=goproxyValue=$(GOPROXY_VALUE) -t $(IMAGE_PREFIX)$(shell basename $(@D)):arm64 $(@D)/ + @for arch in $(ARCHS); do \ + $(SUDO) docker build --platform linux/$$arch --build-arg=revision=$(GIT_REVISION) --build-arg=goproxyValue=$(GOPROXY_VALUE) -t $(IMAGE_PREFIX)$(shell basename $(@D)):$$arch $(@D)/ ; \ + done touch $@ packaging/deb/debian-systemd/$(UPTODATE): packaging/deb/debian-systemd/Dockerfile - $(SUDO) docker build --platform linux/amd64 --build-arg=revision=$(GIT_REVISION) --build-arg=goproxyValue=$(GOPROXY_VALUE) -t $(IMAGE_PREFIX)$(shell basename $(@D)):amd64 $(@D)/ - $(SUDO) docker build --platform linux/arm64 --build-arg=revision=$(GIT_REVISION) --build-arg=goproxyValue=$(GOPROXY_VALUE) -t $(IMAGE_PREFIX)$(shell basename $(@D)):arm64 $(@D)/ + @for arch in $(ARCHS); do \ + $(SUDO) docker build --platform linux/$$arch --build-arg=revision=$(GIT_REVISION) --build-arg=goproxyValue=$(GOPROXY_VALUE) -t $(IMAGE_PREFIX)$(shell basename $(@D)):$$arch $(@D)/ ; \ + done touch $@ .PHONY: test-packages From 35f46e0b68903ac615889db7f9915ae7c5845079 Mon Sep 17 00:00:00 2001 From: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> Date: Mon, 20 Apr 2026 15:49:33 -0700 Subject: [PATCH 13/17] Teting for cgroups v2 Signed-off-by: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> --- tools/packaging/test-packages | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tools/packaging/test-packages b/tools/packaging/test-packages index bc1392499f3..29b27427787 100755 --- a/tools/packaging/test-packages +++ b/tools/packaging/test-packages @@ -48,9 +48,17 @@ function test_with_systemd() { echo "Testing $install_command on $image ($platform)" - container=$(docker run --platform="${platform}" --privileged --tmpfs /run --tmpfs /run/lock -v /sys/fs/cgroup:/sys/fs/cgroup -td -v "$(pwd)"/dist:/opt/cortex -v "$(pwd)"/docs/configuration:/opt/config:ro -p 9009 "${image}") + container=$(docker run --platform="${platform}" --privileged --cgroupns=host --tmpfs /run --tmpfs /run/lock -v /sys/fs/cgroup:/sys/fs/cgroup:rw -td -v "$(pwd)"/dist:/opt/cortex -v "$(pwd)"/docs/configuration:/opt/config:ro -p 9009 "${image}") CONTAINERS+=("${container}") + # Wait briefly for systemd to initialize and verify container is running + sleep 2 + if ! docker inspect --format='{{.State.Running}}' "${container}" | grep -q true; then + echo "Container failed to start:" + docker logs "${container}" || true + error "Container ${container} is not running" + fi + port=$(docker inspect --format='{{(index (index .NetworkSettings.Ports "9009/tcp") 0).HostPort}}' "${container}") docker exec "${container}" /bin/bash -c "${install_command}; mkdir -p /tmp/cortex/tsdb /tmp/cortex/tsdb-sync /tmp/cortex/compactor /tmp/cortex/rules /tmp/cortex/alerts /var/lib/cortex/data/tsdb && chown -R cortex:cortex /tmp/cortex /var/lib/cortex; cp /opt/config/single-process-config-blocks-local.yaml /etc/cortex/single-process-config.yaml && chown root:cortex /etc/cortex/single-process-config.yaml; systemctl start cortex.service; systemctl enable cortex.service" From 4df514857b9d52383b978ce8d60485d10e6a76a8 Mon Sep 17 00:00:00 2001 From: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> Date: Mon, 20 Apr 2026 16:02:07 -0700 Subject: [PATCH 14/17] Fix binary permissions before building packages Signed-off-by: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> --- .github/workflows/test-build-deploy.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test-build-deploy.yml b/.github/workflows/test-build-deploy.yml index 4d00e14af64..10e1ace3132 100644 --- a/.github/workflows/test-build-deploy.yml +++ b/.github/workflows/test-build-deploy.yml @@ -350,6 +350,8 @@ jobs: with: name: Dist Binaries path: dist + - name: Fix Permissions + run: chmod +x dist/cortex-* dist/query-tee-* - name: Build Packages run: | touch dist/.uptodate From b0563e65b04094ee30901cfad86895a2907add97 Mon Sep 17 00:00:00 2001 From: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> Date: Mon, 20 Apr 2026 16:23:40 -0700 Subject: [PATCH 15/17] Split to its own file Signed-off-by: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> --- .github/workflows/distribution.yml | 91 +++++++++++++++++++++++++ .github/workflows/test-build-deploy.yml | 76 --------------------- 2 files changed, 91 insertions(+), 76 deletions(-) create mode 100644 .github/workflows/distribution.yml diff --git a/.github/workflows/distribution.yml b/.github/workflows/distribution.yml new file mode 100644 index 00000000000..20497ce037a --- /dev/null +++ b/.github/workflows/distribution.yml @@ -0,0 +1,91 @@ +name: distribution +permissions: read-all +on: + push: + branches: [master] + tags: + - v[0-9]+.[0-9]+.[0-9]+** # Tag filters not as strict due to different regex system on Github Actions + paths-ignore: + - 'build-image/**' + - '.github/workflows/build-image.yml' + pull_request: + paths-ignore: + - 'build-image/**' + - '.github/workflows/build-image.yml' + +jobs: + build-dist: + runs-on: ubuntu-24.04 + container: + image: quay.io/cortexproject/build-image:master-ee0b97cc37 + steps: + - name: Checkout Repo + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - name: Setup Git safe.directory + run: | + echo "this step is needed because when running in container, actions/checkout does not set safe.directory effectively." + echo "See https://github.com/actions/runner/issues/2033. We should use --system instead of --global" + git config --system --add safe.directory $GITHUB_WORKSPACE + - name: Sym Link Expected Path to Workspace + run: | + mkdir -p /go/src/github.com/cortexproject/cortex + ln -s $GITHUB_WORKSPACE/* /go/src/github.com/cortexproject/cortex + - name: Build Dist Binaries + run: | + touch build-image/.uptodate + make BUILD_IN_CONTAINER=false dist + - name: Upload Dist Binaries + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: Dist Binaries + path: dist/ + + packages: + needs: build-dist + runs-on: ubuntu-24.04 + steps: + - name: Checkout Repo + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - name: Download Dist Binaries + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: Dist Binaries + path: dist + - name: Fix Permissions + run: chmod +x dist/cortex-* dist/query-tee-* + - name: Build Packages + run: | + touch dist/.uptodate + make TTY='' ARCHS=amd64 packages + - name: Upload Packages + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: Packages + path: | + dist/*.deb + dist/*.rpm + dist/*-sha-256 + + test-packages: + needs: packages + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + include: + - runner: ubuntu-24.04 + arch: amd64 + - runner: ubuntu-24.04-arm + arch: arm64 + steps: + - name: Checkout Repo + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - name: Download Packages + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: Packages + path: dist + - name: Build Test Images + run: make TTY='' ARCHS=${{ matrix.arch }} packaging/rpm/rockylinux-systemd/.uptodate packaging/deb/debian-systemd/.uptodate + - name: Test Packages + run: ./tools/packaging/test-packages quay.io/cortexproject/ $(cat VERSION) ${{ matrix.arch }} diff --git a/.github/workflows/test-build-deploy.yml b/.github/workflows/test-build-deploy.yml index 10e1ace3132..a0541ae3bd5 100644 --- a/.github/workflows/test-build-deploy.yml +++ b/.github/workflows/test-build-deploy.yml @@ -156,32 +156,6 @@ jobs: name: Docker Images path: ./images.tar - build-dist: - runs-on: ubuntu-24.04 - container: - image: quay.io/cortexproject/build-image:master-ee0b97cc37 - steps: - - name: Checkout Repo - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - - name: Setup Git safe.directory - run: | - echo "this step is needed because when running in container, actions/checkout does not set safe.directory effectively." - echo "See https://github.com/actions/runner/issues/2033. We should use --system instead of --global" - git config --system --add safe.directory $GITHUB_WORKSPACE - - name: Sym Link Expected Path to Workspace - run: | - mkdir -p /go/src/github.com/cortexproject/cortex - ln -s $GITHUB_WORKSPACE/* /go/src/github.com/cortexproject/cortex - - name: Build Dist Binaries - run: | - touch build-image/.uptodate - make BUILD_IN_CONTAINER=false dist - - name: Upload Dist Binaries - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 - with: - name: Dist Binaries - path: dist/ - integration: needs: [build, lint] runs-on: ${{ matrix.runner }} @@ -339,56 +313,6 @@ jobs: echo "Running configs integration tests on ${{ matrix.arch }}" make BUILD_IMAGE=quay.io/cortexproject/build-image:master-ee0b97cc37 TTY='' configs-integration-test - packages: - needs: build-dist - runs-on: ubuntu-24.04 - steps: - - name: Checkout Repo - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - - name: Download Dist Binaries - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 - with: - name: Dist Binaries - path: dist - - name: Fix Permissions - run: chmod +x dist/cortex-* dist/query-tee-* - - name: Build Packages - run: | - touch dist/.uptodate - make TTY='' ARCHS=amd64 packages - - name: Upload Packages - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 - with: - name: Packages - path: | - dist/*.deb - dist/*.rpm - dist/*-sha-256 - - test-packages: - needs: packages - runs-on: ${{ matrix.runner }} - strategy: - fail-fast: false - matrix: - include: - - runner: ubuntu-24.04 - arch: amd64 - - runner: ubuntu-24.04-arm - arch: arm64 - steps: - - name: Checkout Repo - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - - name: Download Packages - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 - with: - name: Packages - path: dist - - name: Build Test Images - run: make TTY='' ARCHS=${{ matrix.arch }} packaging/rpm/rockylinux-systemd/.uptodate packaging/deb/debian-systemd/.uptodate - - name: Test Packages - run: ./tools/packaging/test-packages quay.io/cortexproject/ $(cat VERSION) ${{ matrix.arch }} - deploy: needs: [build, test, lint, integration, integration-configs-db] if: (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')) && github.repository == 'cortexproject/cortex' From 738eb346d36459fb5cec95443b6b7868c23145dc Mon Sep 17 00:00:00 2001 From: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> Date: Fri, 24 Apr 2026 15:32:02 -0700 Subject: [PATCH 16/17] Implement loop Signed-off-by: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> --- tools/packaging/test-packages | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tools/packaging/test-packages b/tools/packaging/test-packages index 29b27427787..58aae868d4f 100755 --- a/tools/packaging/test-packages +++ b/tools/packaging/test-packages @@ -51,9 +51,17 @@ function test_with_systemd() { container=$(docker run --platform="${platform}" --privileged --cgroupns=host --tmpfs /run --tmpfs /run/lock -v /sys/fs/cgroup:/sys/fs/cgroup:rw -td -v "$(pwd)"/dist:/opt/cortex -v "$(pwd)"/docs/configuration:/opt/config:ro -p 9009 "${image}") CONTAINERS+=("${container}") - # Wait briefly for systemd to initialize and verify container is running - sleep 2 - if ! docker inspect --format='{{.State.Running}}' "${container}" | grep -q true; then + # Wait for container to be running (timeout 10 seconds) + timeout=10 + while [ $timeout -gt 0 ]; do + if docker inspect --format='{{.State.Running}}' "${container}" | grep -q true; then + break + fi + sleep 1 + timeout=$((timeout - 1)) + done + + if [ $timeout -eq 0 ]; then echo "Container failed to start:" docker logs "${container}" || true error "Container ${container} is not running" From dffab04900c3eaf850085e3e3ed1f5565aec9158 Mon Sep 17 00:00:00 2001 From: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> Date: Fri, 24 Apr 2026 16:14:53 -0700 Subject: [PATCH 17/17] Apply only to master Signed-off-by: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> --- .github/workflows/distribution.yml | 7 ------- RELEASE.md | 8 ++++---- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/.github/workflows/distribution.yml b/.github/workflows/distribution.yml index 20497ce037a..5043be459c3 100644 --- a/.github/workflows/distribution.yml +++ b/.github/workflows/distribution.yml @@ -5,13 +5,6 @@ on: branches: [master] tags: - v[0-9]+.[0-9]+.[0-9]+** # Tag filters not as strict due to different regex system on Github Actions - paths-ignore: - - 'build-image/**' - - '.github/workflows/build-image.yml' - pull_request: - paths-ignore: - - 'build-image/**' - - '.github/workflows/build-image.yml' jobs: build-dist: diff --git a/RELEASE.md b/RELEASE.md index ed979cd5abb..0b2714621b5 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -102,8 +102,8 @@ To publish a release candidate: 1. Wait until CI pipeline succeeded (once a tag is created, the release process through GitHub actions will be triggered for this tag) 1. Create a pre-release in GitHub - Write the release notes (including a copy-paste of the changelog) - - Build binaries with `make dist` and attach them to the release - - Build packages with `make packages`, test them with `make test-packages` and attach them to the release + - Download binaries from the `Dist Binaries` artifact in the [distribution](https://github.com/cortexproject/cortex/actions/workflows/distribution.yml) GitHub Actions workflow for the tag and attach them to the release + - Download packages from the `Packages` artifact in the [distribution](https://github.com/cortexproject/cortex/actions/workflows/distribution.yml) GitHub Actions workflow for the tag and attach them to the release 1. [Sign the artifact and generate SBOM for the release](#sing-and-sbom) ### Publish a stable release @@ -119,8 +119,8 @@ To publish a stable release: 1. Wait until CI pipeline succeeded (once a tag is created, the release process through GitHub actions will be triggered for this tag) 1. Create a release in GitHub - Write the release notes (including a copy-paste of the changelog) - - Build binaries with `make dist` and attach them to the release - - Build packages with `make packages`, test them with `make test-packages` and attach them to the release + - Download binaries from the `Dist Binaries` artifact in the [distribution](https://github.com/cortexproject/cortex/actions/workflows/distribution.yml) GitHub Actions workflow for the tag and attach them to the release + - Download packages from the `Packages` artifact in the [distribution](https://github.com/cortexproject/cortex/actions/workflows/distribution.yml) GitHub Actions workflow for the tag and attach them to the release 1. [Sign the artifact and generate SBOM for the release](#sing-and-sbom) 1. Merge the release branch `release-x.y` to `master` - Create `merge-release-X.Y-to-master` branch **from `release-X.Y` branch** locally