From b4ff123db1f57aa7abcf73f2bb442f95efcbe991 Mon Sep 17 00:00:00 2001 From: Tom Martensen Date: Fri, 10 Jul 2026 15:32:37 +0200 Subject: [PATCH 1/4] Increase infra-server readiness probe timeout to 5 seconds The default 1-second timeout was too aggressive for HTTPS probes, causing intermittent failures when TLS handshake or response time exceeded the limit. This resulted in frequent "TLS handshake error ... EOF" log messages and Unhealthy pod events. Effect: Eliminates most readiness probe timeouts and reduces log noise from kubelet probe connections, improving pod stability. Co-Authored-By: Claude Sonnet 4.5 --- chart/infra-server/templates/deployment.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/chart/infra-server/templates/deployment.yaml b/chart/infra-server/templates/deployment.yaml index 548efc08b..b8dfe1d45 100644 --- a/chart/infra-server/templates/deployment.yaml +++ b/chart/infra-server/templates/deployment.yaml @@ -38,6 +38,8 @@ spec: scheme: HTTPS initialDelaySeconds: 5 periodSeconds: 5 + timeoutSeconds: 5 + failureThreshold: 3 command: - /infra-server - -config-dir From 0cddafb6b186238c39d61ca756e4c47cda8b2eb2 Mon Sep 17 00:00:00 2001 From: Tom Martensen Date: Fri, 10 Jul 2026 15:43:49 +0200 Subject: [PATCH 2/4] Add dedicated /healthz endpoint for readiness probes Adds an explicit health check endpoint that bypasses authentication and canonical redirects. This is more robust than relying on User-Agent detection in wrapHealthCheck, which only works after the TLS handshake completes. Changes: - Add /healthz handler in server.go that always returns 200 OK - Update readiness probe to use /healthz instead of / - Increase probe periodSeconds from 5 to 10 to reduce probe frequency - Keep wrapHealthCheck for GCP load balancer health checks (GoogleHC) Effect: Eliminates authentication/redirect overhead from probe requests and provides a more explicit, maintainable health check mechanism. Combined with the increased timeout, this ensures reliable readiness detection without TLS handshake errors. Co-Authored-By: Claude Sonnet 4.5 --- chart/infra-server/templates/deployment.yaml | 4 ++-- pkg/server/server.go | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/chart/infra-server/templates/deployment.yaml b/chart/infra-server/templates/deployment.yaml index b8dfe1d45..16c602062 100644 --- a/chart/infra-server/templates/deployment.yaml +++ b/chart/infra-server/templates/deployment.yaml @@ -33,11 +33,11 @@ spec: value: "{{ .Values.testMode }}" readinessProbe: httpGet: - path: / + path: /healthz port: 8443 scheme: HTTPS initialDelaySeconds: 5 - periodSeconds: 5 + periodSeconds: 10 timeoutSeconds: 5 failureThreshold: 3 command: diff --git a/pkg/server/server.go b/pkg/server/server.go index 0285f5e02..464a0b7ef 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -171,6 +171,12 @@ func (s *server) RunServer() (<-chan error, error) { routeMux.Handle("/v1/", gwMux) s.oidc.Handle(routeMux) + // Dedicated health endpoint for Kubernetes readiness probes. + // Bypasses authentication and redirects to prevent probe timeouts. + mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }) + mux.Handle("/", wrapHealthCheck( wrapCanonicalRedirect( From 5e98a34a3cad45daf513618385d17a9204a534a9 Mon Sep 17 00:00:00 2001 From: Tom Martensen Date: Fri, 10 Jul 2026 16:23:11 +0200 Subject: [PATCH 3/4] fix lint issue --- pkg/server/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/server/server.go b/pkg/server/server.go index 464a0b7ef..fabc0d8bb 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -173,7 +173,7 @@ func (s *server) RunServer() (<-chan error, error) { // Dedicated health endpoint for Kubernetes readiness probes. // Bypasses authentication and redirects to prevent probe timeouts. - mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { + mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }) From 9090c2abf36d57dfc83777f2637c06d54ed38a5e Mon Sep 17 00:00:00 2001 From: Tom Martensen Date: Sat, 11 Jul 2026 08:44:47 +0200 Subject: [PATCH 4/4] switch to stable test image --- .github/workflows/PR.yaml | 4 ++-- .github/workflows/build-and-push.yaml | 4 ++-- .github/workflows/lint.yaml | 2 +- .github/workflows/unit-tests.yaml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/PR.yaml b/.github/workflows/PR.yaml index d8cf388c7..ef0e2ab84 100644 --- a/.github/workflows/PR.yaml +++ b/.github/workflows/PR.yaml @@ -45,7 +45,7 @@ jobs: - create-dev-cluster runs-on: ubuntu-latest container: - image: quay.io/stackrox-io/apollo-ci:stackrox-test-0.4.9 + image: quay.io/stackrox-io/apollo-ci:stackrox-test-stable steps: - name: Checkout @@ -65,7 +65,7 @@ jobs: - create-dev-cluster runs-on: ubuntu-latest container: - image: quay.io/stackrox-io/apollo-ci:stackrox-test-0.4.9 + image: quay.io/stackrox-io/apollo-ci:stackrox-test-stable env: KUBECONFIG: /github/home/artifacts/kubeconfig INFRA_TOKEN: ${{ secrets.INFRA_TOKEN }} diff --git a/.github/workflows/build-and-push.yaml b/.github/workflows/build-and-push.yaml index 5f0103d79..00e61df30 100644 --- a/.github/workflows/build-and-push.yaml +++ b/.github/workflows/build-and-push.yaml @@ -11,7 +11,7 @@ jobs: build-and-push-server: runs-on: ubuntu-latest container: - image: quay.io/stackrox-io/apollo-ci:stackrox-test-0.4.4 + image: quay.io/stackrox-io/apollo-ci:stackrox-test-stable steps: - name: Checkout @@ -53,7 +53,7 @@ jobs: build-and-push-certifier: runs-on: ubuntu-latest container: - image: quay.io/stackrox-io/apollo-ci:stackrox-test-0.4.4 + image: quay.io/stackrox-io/apollo-ci:stackrox-test-stable steps: - name: Checkout diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 46cbab42f..982a1b852 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -37,7 +37,7 @@ jobs: shellcheck: runs-on: ubuntu-latest container: - image: quay.io/stackrox-io/apollo-ci:stackrox-test-0.4.4 + image: quay.io/stackrox-io/apollo-ci:stackrox-test-stable steps: - name: Checkout uses: actions/checkout@v7 diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index 18d7e049b..4879c51c7 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -11,7 +11,7 @@ jobs: unit-test: runs-on: ubuntu-latest container: - image: quay.io/stackrox-io/apollo-ci:stackrox-test-0.4.4 + image: quay.io/stackrox-io/apollo-ci:stackrox-test-stable steps: - name: Checkout