From 5cfab48b5b78d005f73b9172fd11ee3ebd040ff8 Mon Sep 17 00:00:00 2001 From: Thomas Binu Thomas Date: Fri, 22 May 2026 12:04:04 +0100 Subject: [PATCH 1/7] feat(charts): add auth-daemon-config secret to pollux --- .../charts/secrets/templates/auth-daemon.yaml | 16 ++++++++++++++++ charts/workflows-cluster/staging-values.yaml | 1 + 2 files changed, 17 insertions(+) create mode 100644 charts/workflows-cluster/charts/secrets/templates/auth-daemon.yaml diff --git a/charts/workflows-cluster/charts/secrets/templates/auth-daemon.yaml b/charts/workflows-cluster/charts/secrets/templates/auth-daemon.yaml new file mode 100644 index 000000000..1ce1dcf02 --- /dev/null +++ b/charts/workflows-cluster/charts/secrets/templates/auth-daemon.yaml @@ -0,0 +1,16 @@ +{{- if eq .Values.cluster "argus" }} +{{- else if eq .Values.cluster "pollux" }} +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + name: auth-daemon-config + namespace: workflows +spec: + encryptedData: + config.yaml: AgCh9l99Vb3PxDSCwApnc5mJO2HnI9W+Y3ENS7y1nTXvzeAnBuasjQmJO8tRgC2Es/BBhqYr6/pGCiU7KTTx5wNxoue4gMc6O/LvJhoc2KjLvdadXSuGPyTIXcBSDuFl4ErrbJXX3fXmSVAFTJtLkdcoMvaMq2zeqOp/dNWAE6slvjDF49BoLfy8LOo1MwIz/8r6DLD6Qpy5SUjlvoHL6ZBVNPlM3jXIcaooHaWzdq+tC/tBg6Za+4sjEzownWwl9pQBx1Ru/4kYnfL5aNQ/olQLKNnNdC3c6i0dSNB7VZHMqTmQIsRdA2WU/QSPBoVKQLeIBjcw1HNRyMxvuG9ojZ/NsgdXrvd6Nrn4t6nYB1rfUV17syhTtsD9iBK/LVkUPr+KMHX9LcE7B66cfhKrHCYM747bM2cI4Qz6ADG2vZ8enMFfCvq2RcnAYqf9pASScwLsCDNRS72ub0t6/j/8TwxSAy1D1LKaZF+1IGsOzFAhIApA5Lu5pYRqg8qwSToEvWcOIodrIkNDpcJuJWChRiV6Y2JJggKvWOSIBLEWxLd1Od7NjpZcRSfmANhHyfctGBXzu++IcYCcauiGm4vIPZjjvsS1fhTKi7rJ1UOiFA3H/jKuew1ZDn5/CGut48hSqvMQWR7ZYgzUnTycUGCs0eX5XJJ/e778dYIKi2pEfArQnXfNWVSgaT43F/4YJAzPodcE/Md9Nncf/has1BFMafliVDbTVEi9Bb66Uhrlinn1RAaMtNzQ1t4ZGGe6EgtuwgycbyKuKCS1rkApamhh8/+N3bi3NJFCfZn5VTyt0r1qRRfU6Ldapn9+deLE5KQRY64Y/nhW96mdimVY78/U2zdSGtbgADox5HQVIWblD/MzpaYytswW5Ik01JulC1mSP5z6GpDhNZG8q82yLNNtI3glPhHKaLybeOzqOq/lCoDxkQ+S3G0q2l22+wgwJcrv5bM08Q8Ug0uFQZDZT41FctHKxtI4s51mvN0tlS5CmG818HBxG4FmoK4hNV7Qal0ioesUGYNFspz1pLrjl3MNv/wKDvwUWwvbkMG9GYe4doL97QLsWANd5YJNTgA+lqCH3i5vjQRgJuS9uOwgCmy1XjNnujIiiHVY6jzpogjL1B2LXG+ebdYdWlmOr5y+5IOQkkVhpLmrfych/GV3A/+aJJAbogbproUJtnm4ySPWKj4zWULm2h3DuF66DgNp1tLmX0ARduKHGgTD1ixgpLGPxdjIkPew+n6tVwSlyRik42B3lBY3aIdiTtIwEuxp1GPXlC6KHtNPBR45MeVYMUVjJsVtB9dLswZ5BUaozvSOsap85YEKO1foSMJliW1KoMYg20UFbLmA0ZrnDDaF8mwOSen6qCKmSrj4eG/rfFDgYe8Wa7N1AIL5IYSS1jgfbDyCt18opMTeNGr7nTyfqzPQSRaEAC9yJ1Vj0QhJ59ZfUQmAGOLKFlEd2HPmxVpIr8Dz/hJJQC30yT2Wf0ekHg== + template: + metadata: + name: auth-daemon-config + namespace: workflows +{{- else }} +{{- end }} diff --git a/charts/workflows-cluster/staging-values.yaml b/charts/workflows-cluster/staging-values.yaml index 49a51f14f..2a04ab1de 100644 --- a/charts/workflows-cluster/staging-values.yaml +++ b/charts/workflows-cluster/staging-values.yaml @@ -83,6 +83,7 @@ vcluster: "/artifact-s3-secret": "graph-proxy/artifact-s3-secret" "/s3-artifact": "workflows/artifact-s3" "/oidc-bff-config": "workflows/oidc-bff-config" + "/auth-daemon-config": "workflows/auth-daemon-config" "/postgres-passwords": "workflows/postgres-passwords" "/postgres-argo-workflows-password": "workflows/postgres-argo-workflows-password" "/graph-proxy-k6-auth": "k6/graph-proxy-k6-auth" From 8e3400429c32673df55e0525e24c9692c0bed243 Mon Sep 17 00:00:00 2001 From: Thomas Binu Thomas Date: Fri, 22 May 2026 12:09:12 +0100 Subject: [PATCH 2/7] feat(charts): add auth-daemon clusterpolicies for copying and security --- .../auth-daemon-config-clusterpolicy.yaml | 128 ++++++++++++++++++ .../templates/sessionspace-clusterpolicy.yaml | 27 ++++ .../synchronize-auth-daemon-config.yaml | 63 +++++++++ 3 files changed, 218 insertions(+) create mode 100644 charts/workflows/templates/auth-daemon-config-clusterpolicy.yaml create mode 100644 charts/workflows/templates/synchronize-auth-daemon-config.yaml diff --git a/charts/workflows/templates/auth-daemon-config-clusterpolicy.yaml b/charts/workflows/templates/auth-daemon-config-clusterpolicy.yaml new file mode 100644 index 000000000..0e9ceb041 --- /dev/null +++ b/charts/workflows/templates/auth-daemon-config-clusterpolicy.yaml @@ -0,0 +1,128 @@ +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: {{ .Release.Name }}-auth-daemon-config +spec: + validationFailureAction: Enforce + background: false + rules: + - name: prohibit-auth-daemon-config-volume + match: + resources: + kinds: + - Pod + namespaceSelector: + matchLabels: + app.kubernetes.io/managed-by: sessionspaces + validate: + message: "auth-daemon-config can only be used by the auth-daemon container" + pattern: + spec: + =(volumes): + - (name): "!auth-daemon-config" + =(secret): + secretName: "!auth-daemon-config" + =(projected): + =(sources): + - =(secret): + name: "!auth-daemon-config" + - name: prohibit-auth-daemon-config-env + match: + resources: + kinds: + - Pod + namespaceSelector: + matchLabels: + app.kubernetes.io/managed-by: sessionspaces + validate: + message: "auth-daemon-config can only be used by the auth-daemon container" + pattern: + spec: + containers: + - (image): "!ghcr.io/diamondlightsource/workflows-auth-daemon:*" + =(env): + - =(valueFrom): + =(secretKeyRef): + name: "!auth-daemon-config" + =(envFrom): + - =(secretRef): + name: "!auth-daemon-config" + =(initContainers): + - (name): "!init" + =(env): + - =(valueFrom): + =(secretKeyRef): + name: "!auth-daemon-config" + =(envFrom): + - =(secretRef): + name: "!auth-daemon-config" + =(ephemeralContainers): + - name: "*" + =(env): + - =(valueFrom): + =(secretKeyRef): + name: "!auth-daemon-config" + =(envFrom): + - =(secretRef): + name: "!auth-daemon-config" + - name: prohibit-auth-daemon-config-volumemounts + match: + resources: + kinds: + - Pod + namespaceSelector: + matchLabels: + app.kubernetes.io/managed-by: sessionspaces + validate: + message: "auth-daemon-config can only be used by the auth-daemon container" + foreach: + - list: "request.object.spec.containers" + deny: + conditions: + all: + - key: '{{ "{{" }} element.name {{ "}}" }}' + operator: NotEquals + value: "wait" + - key: '{{ "{{" }} contains(element.image, ''ghcr.io/diamondlightsource/workflows-auth-daemon'') {{ "}}" }}' + operator: Equals + value: false + - key: '{{ "{{" }} (element.volumeMounts || `[]`)[?name == ''auth-daemon-config''] | length(@) {{ "}}" }}' + operator: GreaterThan + value: 0 + - list: "request.object.spec.initContainers || []" + deny: + conditions: + all: + - key: '{{ "{{" }} element.name {{ "}}" }}' + operator: NotEquals + value: "init" + - key: '{{ "{{" }} (element.volumeMounts || `[]`)[?name == ''auth-daemon-config''] | length(@) {{ "}}" }}' + operator: GreaterThan + value: 0 + - list: "request.object.spec.ephemeralContainers || []" + deny: + conditions: + all: + - key: '{{ "{{" }} (element.volumeMounts || `[]`)[?name == ''auth-daemon-config''] | length(@) {{ "}}" }}' + operator: GreaterThan + value: 0 + - name: block-exec-into-auth-daemon-pods + match: + resources: + kinds: + - Pod/exec + namespaceSelector: + matchLabels: + app.kubernetes.io/managed-by: sessionspaces + context: + - name: podContainers + apiCall: + urlPath: '/api/v1/namespaces/{{ "{{" }}request.namespace{{ "}}" }}/pods/{{ "{{" }}request.name{{ "}}" }}' + jmesPath: "spec.containers" + validate: + deny: + conditions: + any: + - key: '{{ "{{" }} podContainers[?contains(image, ''ghcr.io/diamondlightsource/workflows-auth-daemon'')] | length(@) {{ "}}" }}' + operator: GreaterThan + value: 0 diff --git a/charts/workflows/templates/sessionspace-clusterpolicy.yaml b/charts/workflows/templates/sessionspace-clusterpolicy.yaml index 826e95cb8..c3317fecf 100644 --- a/charts/workflows/templates/sessionspace-clusterpolicy.yaml +++ b/charts/workflows/templates/sessionspace-clusterpolicy.yaml @@ -142,3 +142,30 @@ spec: expression: resource.Get("v1", "secrets", "workflows", "artifact-s3") generate: - expression: generator.Apply(variables.targetNs, [variables.sourceSecret]) +--- +apiVersion: policies.kyverno.io/v1alpha1 +kind: GeneratingPolicy +metadata: + name: copy-auth-daemon-config +spec: + evaluation: + generateExisting: + enabled: false + synchronize: + enabled: false + matchConstraints: + resourceRules: + - apiGroups: [""] + apiVersions: ["v1"] + operations: ["CREATE"] + resources: ["namespaces"] + namespaceSelector: + matchLabels: + app.kubernetes.io/managed-by: sessionspaces + variables: + - name: targetNs + expression: "object.metadata.name" + - name: sourceSecret + expression: resource.Get("v1", "secrets", "workflows", "auth-daemon-config") + generate: + - expression: generator.Apply(variables.targetNs, [variables.sourceSecret]) diff --git a/charts/workflows/templates/synchronize-auth-daemon-config.yaml b/charts/workflows/templates/synchronize-auth-daemon-config.yaml new file mode 100644 index 000000000..9a4c66cb4 --- /dev/null +++ b/charts/workflows/templates/synchronize-auth-daemon-config.yaml @@ -0,0 +1,63 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: auth-daemon-config-cloner + namespace: workflows +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: auth-daemon-config-cloner +rules: + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["list", "get", "watch"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["create", "update", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: auth-daemon-config-cloner +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: auth-daemon-config-cloner +subjects: + - kind: ServiceAccount + name: auth-daemon-config-cloner + namespace: workflows +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: auth-daemon-config-cloner + namespace: workflows +rules: + - apiGroups: [""] + resources: ["secrets"] + resourceNames: ["auth-daemon-config"] + verbs: ["get"] + - apiGroups: ["batch"] + resources: ["cronjobs"] + verbs: ["get"] + - apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["create"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: auth-daemon-config-cloner + namespace: workflows +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: auth-daemon-config-cloner +subjects: + - kind: ServiceAccount + name: auth-daemon-config-cloner + namespace: workflows + From ed02ffae505555b25a3078221e977a4fc0676708 Mon Sep 17 00:00:00 2001 From: Thomas Binu Thomas Date: Fri, 22 May 2026 12:18:59 +0100 Subject: [PATCH 3/7] feat(charts): add auth-daemon secret copying cron job as fallback --- .../synchronize-auth-daemon-config.yaml | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/charts/workflows/templates/synchronize-auth-daemon-config.yaml b/charts/workflows/templates/synchronize-auth-daemon-config.yaml index 9a4c66cb4..dac783c3f 100644 --- a/charts/workflows/templates/synchronize-auth-daemon-config.yaml +++ b/charts/workflows/templates/synchronize-auth-daemon-config.yaml @@ -60,4 +60,73 @@ subjects: - kind: ServiceAccount name: auth-daemon-config-cloner namespace: workflows +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: copy-auth-daemon-config + namespace: workflows +spec: + # Suspended: intended to be triggered manually on install and after secret rotation. + # eg kubectl create job --from=cronjob/copy-auth-daemon-config -nworkflows copy-auth-daemon-config-$(date +%s) + suspend: true + schedule: "0 3 * * *" + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + backoffLimit: 0 + ttlSecondsAfterFinished: 300 + template: + spec: + serviceAccountName: auth-daemon-config-cloner + restartPolicy: Never + containers: + - name: sync + image: alpine/kubectl:1.34.2 + imagePullPolicy: IfNotPresent + command: ["/bin/sh", "-c"] + args: + - | + set -euo pipefail + + SOURCE_NAMESPACE="${SOURCE_NAMESPACE:-workflows}" + SECRET_NAME="${SECRET_NAME:-auth-daemon-config}" + LABEL_SELECTOR="${LABEL_SELECTOR:-app.kubernetes.io/managed-by=sessionspaces}" + + echo "Starting sync of ${SOURCE_NAMESPACE}/${SECRET_NAME} to namespaces with ${LABEL_SELECTOR}" + + if ! kubectl -n "${SOURCE_NAMESPACE}" get secret "${SECRET_NAME}" >/dev/null 2>&1; then + echo "ERROR: source secret ${SOURCE_NAMESPACE}/${SECRET_NAME} not found" >&2 + exit 1 + fi + + CONFIG_B64="$(kubectl -n "${SOURCE_NAMESPACE}" get secret "${SECRET_NAME}" -o jsonpath='{.data.config\.yaml}' || true)" + + if [ -z "${CONFIG_B64}" ]; then + echo "ERROR: expected key 'config.yaml' missing from secret" >&2 + exit 2 + fi + + for ns in $(kubectl get ns -l "${LABEL_SELECTOR}" -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do + echo "Syncing into namespace: ${ns}" + kubectl apply --server-side --force-conflicts -f - < Date: Thu, 28 May 2026 17:31:45 +0100 Subject: [PATCH 4/7] feat(charts): auth-daemon-config secret clone chainsaw test --- .../chainsaw-test.yaml | 137 ++++++++++++++++++ 1 file changed, 137 insertions(+) create mode 100644 charts/workflows/test-policy/auth-daemon-config-clone/chainsaw-test.yaml diff --git a/charts/workflows/test-policy/auth-daemon-config-clone/chainsaw-test.yaml b/charts/workflows/test-policy/auth-daemon-config-clone/chainsaw-test.yaml new file mode 100644 index 000000000..ae52140b8 --- /dev/null +++ b/charts/workflows/test-policy/auth-daemon-config-clone/chainsaw-test.yaml @@ -0,0 +1,137 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: auth-daemon-config-clone +spec: + concurrent: false + steps: + - try: + - apply: + resource: + apiVersion: v1 + kind: Namespace + metadata: + name: workflows + - apply: + resource: + apiVersion: v1 + kind: Secret + metadata: + name: auth-daemon-config + namespace: workflows + data: + config.yaml: Y29uZmlnOiB2YWx1ZQ== + - apply: + resource: + apiVersion: v1 + kind: Namespace + metadata: + name: auth-daemon-session + labels: + app.kubernetes.io/managed-by: sessionspaces + - assert: + resource: + apiVersion: v1 + kind: Secret + metadata: + name: auth-daemon-config + namespace: auth-daemon-session +--- +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: auth-daemon-config-clone-via-cronjob +spec: + concurrent: false + steps: + - try: + - apply: + resource: + apiVersion: v1 + kind: Namespace + metadata: + name: workflows + - apply: + resource: + apiVersion: v1 + kind: Secret + metadata: + name: auth-daemon-config + namespace: workflows + type: Opaque + data: + config.yaml: Y29uZmlnOiB2YWx1ZQ== + - apply: + resource: + apiVersion: v1 + kind: Namespace + metadata: + name: auth-daemon-session-cronjob + labels: + app.kubernetes.io/managed-by: sessionspaces + # kyverno generating policy should create the initial secret + - assert: + resource: + apiVersion: v1 + kind: Secret + metadata: + name: auth-daemon-config + namespace: auth-daemon-session-cronjob + data: + config.yaml: Y29uZmlnOiB2YWx1ZQ== + # update the secret + - apply: + resource: + apiVersion: v1 + kind: Secret + metadata: + name: auth-daemon-config + namespace: workflows + data: + config.yaml: dXBkYXRlZC1jb25maWc= + # confirm that the kyverno policy does not sync the updated secret + - sleep: + duration: 5s + - assert: + resource: + apiVersion: v1 + kind: Secret + metadata: + name: auth-daemon-config + namespace: auth-daemon-session-cronjob + data: + config.yaml: Y29uZmlnOiB2YWx1ZQ== + # apply the CronJob and associated RBAC + - apply: + file: ../../templates/synchronize-auth-daemon-config.yaml + - assert: + resource: + apiVersion: batch/v1 + kind: CronJob + metadata: + name: copy-auth-daemon-config + namespace: workflows + # run job and wait for it to complete + - script: + content: | + kubectl -n workflows create job --from=cronjob/copy-auth-daemon-config copy-auth-daemon-config-manual + - wait: + apiVersion: batch/v1 + kind: Job + namespace: workflows + name: copy-auth-daemon-config-manual + timeout: 2m + for: + condition: + name: Complete + value: "True" + # confirm that the Secret has been updated + - assert: + resource: + apiVersion: v1 + kind: Secret + metadata: + name: auth-daemon-config + namespace: auth-daemon-session-cronjob + data: + config.yaml: dXBkYXRlZC1jb25maWc= From 5a6f53dc242ade758a6f9e2e6459a431db7f8fcb Mon Sep 17 00:00:00 2001 From: Thomas Binu Thomas Date: Thu, 28 May 2026 17:41:09 +0100 Subject: [PATCH 5/7] feat(charts): verbose timeout setup for kyverno chainsaw tests --- .../test-policy/pod-securitycontext/chainsaw-test.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/charts/sessionspaces/test-policy/pod-securitycontext/chainsaw-test.yaml b/charts/sessionspaces/test-policy/pod-securitycontext/chainsaw-test.yaml index 912594695..5aecac2cf 100644 --- a/charts/sessionspaces/test-policy/pod-securitycontext/chainsaw-test.yaml +++ b/charts/sessionspaces/test-policy/pod-securitycontext/chainsaw-test.yaml @@ -3,6 +3,13 @@ kind: Test metadata: name: pod-securitycontext spec: + timeouts: + apply: 45s + assert: 30s + cleanup: 2m + delete: 1m + error: 30s + exec: 15s namespaceTemplate: metadata: labels: From b62a584610c7e6921b9e02be7d7b80eb75a73e69 Mon Sep 17 00:00:00 2001 From: Thomas Binu Thomas Date: Fri, 22 May 2026 12:20:17 +0100 Subject: [PATCH 6/7] feat(charts): updated chart versions --- charts/sessionspaces/Chart.yaml | 2 +- charts/workflows-cluster/Chart.yaml | 2 +- charts/workflows/Chart.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/sessionspaces/Chart.yaml b/charts/sessionspaces/Chart.yaml index 159b94b60..27d222e54 100644 --- a/charts/sessionspaces/Chart.yaml +++ b/charts/sessionspaces/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: sessionspaces description: Namespace controller for creating session namespaces type: application -version: 0.3.28 +version: 0.3.29 appVersion: 0.1.6 dependencies: - name: common diff --git a/charts/workflows-cluster/Chart.yaml b/charts/workflows-cluster/Chart.yaml index f0de247de..871874a16 100644 --- a/charts/workflows-cluster/Chart.yaml +++ b/charts/workflows-cluster/Chart.yaml @@ -3,7 +3,7 @@ name: workflows-cluster description: A virtual cluster for Data Analysis workflows type: application -version: 0.13.7 +version: 0.13.8 dependencies: - name: common version: 2.23.0 diff --git a/charts/workflows/Chart.yaml b/charts/workflows/Chart.yaml index 9b0541b71..fb85be88d 100644 --- a/charts/workflows/Chart.yaml +++ b/charts/workflows/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: workflows description: Data Analysis workflow orchestration type: application -version: 0.13.47 +version: 0.13.48 dependencies: - name: argo-workflows repository: https://argoproj.github.io/argo-helm From 55bef89f35b5770ef1950ce9724ff329acbb178b Mon Sep 17 00:00:00 2001 From: TBThomas56 Date: Fri, 29 May 2026 13:14:59 +0100 Subject: [PATCH 7/7] fix(ci): simplified MPI job --- .github/workflows/_kyverno_policy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_kyverno_policy.yaml b/.github/workflows/_kyverno_policy.yaml index 62d6fb155..e60be3371 100644 --- a/.github/workflows/_kyverno_policy.yaml +++ b/.github/workflows/_kyverno_policy.yaml @@ -51,7 +51,7 @@ jobs: - name: Install mpi-operator CRDs run: | - kubectl apply --server-side -f https://raw.githubusercontent.com/kubeflow/mpi-operator/v0.7.0/deploy/v2beta1/mpi-operator.yaml + curl -sL https://raw.githubusercontent.com/kubeflow/mpi-operator/v0.7.0/deploy/v2beta1/mpi-operator.yaml | yq e 'select(.kind == "CustomResourceDefinition")' | kubectl apply --server-side -f - - name: Deploy and wait for Policies, ClusterRoles, and ClusterRoleBindings run: |