Tbt/auth daemon config secret

.github/workflows/_kyverno_policy.yaml

@@ -51,7 +51,7 @@ jobs:
 
       - name: Install mpi-operator CRDs
         run: |
-          kubectl apply --server-side -f https://raw.githubusercontent.com/kubeflow/mpi-operator/v0.7.0/deploy/v2beta1/mpi-operator.yaml
+          curl -sL https://raw.githubusercontent.com/kubeflow/mpi-operator/v0.7.0/deploy/v2beta1/mpi-operator.yaml | yq e 'select(.kind == "CustomResourceDefinition")' | kubectl apply --server-side -f -
 
       - name: Deploy and wait for Policies, ClusterRoles, and ClusterRoleBindings
         run: |

charts/sessionspaces/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: sessionspaces
 description: Namespace controller for creating session namespaces
 type: application
-version: 0.3.28
+version: 0.3.29
 appVersion: 0.1.6
 dependencies:
   - name: common

charts/sessionspaces/test-policy/pod-securitycontext/chainsaw-test.yaml

@@ -3,6 +3,13 @@ kind: Test
 metadata:
   name: pod-securitycontext
 spec:
+  timeouts:
+    apply: 45s
+    assert: 30s
+    cleanup: 2m
+    delete: 1m
+    error: 30s
+    exec: 15s
   namespaceTemplate:
     metadata:
       labels:

charts/workflows-cluster/Chart.yaml

@@ -3,7 +3,7 @@ name: workflows-cluster
 description: A virtual cluster for Data Analysis workflows
 type: application
 
-version: 0.13.7
+version: 0.13.8
 dependencies:
   - name: common
     version: 2.23.0

charts/workflows-cluster/charts/secrets/templates/auth-daemon.yaml

@@ -0,0 +1,16 @@
+{{- if eq .Values.cluster "argus" }}
+{{- else if eq .Values.cluster "pollux" }}
+apiVersion: bitnami.com/v1alpha1
+kind: SealedSecret
+metadata:
+  name: auth-daemon-config
+  namespace: workflows
+spec:
+  encryptedData:
+    config.yaml: AgCh9l99Vb3PxDSCwApnc5mJO2HnI9W+Y3ENS7y1nTXvzeAnBuasjQmJO8tRgC2Es/BBhqYr6/pGCiU7KTTx5wNxoue4gMc6O/LvJhoc2KjLvdadXSuGPyTIXcBSDuFl4ErrbJXX3fXmSVAFTJtLkdcoMvaMq2zeqOp/dNWAE6slvjDF49BoLfy8LOo1MwIz/8r6DLD6Qpy5SUjlvoHL6ZBVNPlM3jXIcaooHaWzdq+tC/tBg6Za+4sjEzownWwl9pQBx1Ru/4kYnfL5aNQ/olQLKNnNdC3c6i0dSNB7VZHMqTmQIsRdA2WU/QSPBoVKQLeIBjcw1HNRyMxvuG9ojZ/NsgdXrvd6Nrn4t6nYB1rfUV17syhTtsD9iBK/LVkUPr+KMHX9LcE7B66cfhKrHCYM747bM2cI4Qz6ADG2vZ8enMFfCvq2RcnAYqf9pASScwLsCDNRS72ub0t6/j/8TwxSAy1D1LKaZF+1IGsOzFAhIApA5Lu5pYRqg8qwSToEvWcOIodrIkNDpcJuJWChRiV6Y2JJggKvWOSIBLEWxLd1Od7NjpZcRSfmANhHyfctGBXzu++IcYCcauiGm4vIPZjjvsS1fhTKi7rJ1UOiFA3H/jKuew1ZDn5/CGut48hSqvMQWR7ZYgzUnTycUGCs0eX5XJJ/e778dYIKi2pEfArQnXfNWVSgaT43F/4YJAzPodcE/Md9Nncf/has1BFMafliVDbTVEi9Bb66Uhrlinn1RAaMtNzQ1t4ZGGe6EgtuwgycbyKuKCS1rkApamhh8/+N3bi3NJFCfZn5VTyt0r1qRRfU6Ldapn9+deLE5KQRY64Y/nhW96mdimVY78/U2zdSGtbgADox5HQVIWblD/MzpaYytswW5Ik01JulC1mSP5z6GpDhNZG8q82yLNNtI3glPhHKaLybeOzqOq/lCoDxkQ+S3G0q2l22+wgwJcrv5bM08Q8Ug0uFQZDZT41FctHKxtI4s51mvN0tlS5CmG818HBxG4FmoK4hNV7Qal0ioesUGYNFspz1pLrjl3MNv/wKDvwUWwvbkMG9GYe4doL97QLsWANd5YJNTgA+lqCH3i5vjQRgJuS9uOwgCmy1XjNnujIiiHVY6jzpogjL1B2LXG+ebdYdWlmOr5y+5IOQkkVhpLmrfych/GV3A/+aJJAbogbproUJtnm4ySPWKj4zWULm2h3DuF66DgNp1tLmX0ARduKHGgTD1ixgpLGPxdjIkPew+n6tVwSlyRik42B3lBY3aIdiTtIwEuxp1GPXlC6KHtNPBR45MeVYMUVjJsVtB9dLswZ5BUaozvSOsap85YEKO1foSMJliW1KoMYg20UFbLmA0ZrnDDaF8mwOSen6qCKmSrj4eG/rfFDgYe8Wa7N1AIL5IYSS1jgfbDyCt18opMTeNGr7nTyfqzPQSRaEAC9yJ1Vj0QhJ59ZfUQmAGOLKFlEd2HPmxVpIr8Dz/hJJQC30yT2Wf0ekHg==
+  template:
+    metadata:
+      name: auth-daemon-config
+      namespace: workflows
+{{- else }}
+{{- end }}

charts/workflows-cluster/staging-values.yaml

@@ -83,6 +83,7 @@ vcluster:
             "/artifact-s3-secret": "graph-proxy/artifact-s3-secret"
             "/s3-artifact": "workflows/artifact-s3"
             "/oidc-bff-config": "workflows/oidc-bff-config"
+            "/auth-daemon-config": "workflows/auth-daemon-config"
             "/postgres-passwords": "workflows/postgres-passwords"
             "/postgres-argo-workflows-password": "workflows/postgres-argo-workflows-password"
             "/graph-proxy-k6-auth": "k6/graph-proxy-k6-auth"

charts/workflows/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: workflows
 description: Data Analysis workflow orchestration
 type: application
-version: 0.13.47
+version: 0.13.48
 dependencies:
   - name: argo-workflows
     repository: https://argoproj.github.io/argo-helm

charts/workflows/templates/auth-daemon-config-clusterpolicy.yaml

@@ -0,0 +1,128 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: {{ .Release.Name }}-auth-daemon-config
+spec:
+  validationFailureAction: Enforce
+  background: false
+  rules:
+    - name: prohibit-auth-daemon-config-volume
+      match:
+        resources:
+          kinds:
+            - Pod
+          namespaceSelector:
+            matchLabels:
+              app.kubernetes.io/managed-by: sessionspaces
+      validate:
+        message: "auth-daemon-config can only be used by the auth-daemon container"
+        pattern:
+          spec:
+            =(volumes):
+              - (name): "!auth-daemon-config"
+                =(secret):
+                  secretName: "!auth-daemon-config"
+                =(projected):
+                  =(sources):
+                    - =(secret):
+                        name: "!auth-daemon-config"
+    - name: prohibit-auth-daemon-config-env
+      match:
+        resources:
+          kinds:
+            - Pod
+          namespaceSelector:
+            matchLabels:
+              app.kubernetes.io/managed-by: sessionspaces
+      validate:
+        message: "auth-daemon-config can only be used by the auth-daemon container"
+        pattern:
+          spec:
+            containers:
+              - (image): "!ghcr.io/diamondlightsource/workflows-auth-daemon:*"
+                =(env):
+                  - =(valueFrom):
+                      =(secretKeyRef):
+                        name: "!auth-daemon-config"
+                =(envFrom):
+                  - =(secretRef):
+                      name: "!auth-daemon-config"
+            =(initContainers):
+              - (name): "!init"
+                =(env):
+                  - =(valueFrom):
+                      =(secretKeyRef):
+                        name: "!auth-daemon-config"
+                =(envFrom):
+                  - =(secretRef):
+                      name: "!auth-daemon-config"
+            =(ephemeralContainers):
+              - name: "*"
+                =(env):
+                  - =(valueFrom):
+                      =(secretKeyRef):
+                        name: "!auth-daemon-config"
+                =(envFrom):
+                  - =(secretRef):
+                      name: "!auth-daemon-config"
+    - name: prohibit-auth-daemon-config-volumemounts
+      match:
+        resources:
+          kinds:
+            - Pod
+          namespaceSelector:
+            matchLabels:
+              app.kubernetes.io/managed-by: sessionspaces
+      validate:
+        message: "auth-daemon-config can only be used by the auth-daemon container"
+        foreach:
+          - list: "request.object.spec.containers"
+            deny:
+              conditions:
+                all:
+                  - key: '{{ "{{" }} element.name {{ "}}" }}'
+                    operator: NotEquals
+                    value: "wait"
+                  - key: '{{ "{{" }} contains(element.image, ''ghcr.io/diamondlightsource/workflows-auth-daemon'') {{ "}}" }}'
+                    operator: Equals
+                    value: false
+                  - key: '{{ "{{" }} (element.volumeMounts || `[]`)[?name == ''auth-daemon-config''] | length(@) {{ "}}" }}'
+                    operator: GreaterThan
+                    value: 0
+          - list: "request.object.spec.initContainers || []"
+            deny:
+              conditions:
+                all:
+                  - key: '{{ "{{" }} element.name {{ "}}" }}'
+                    operator: NotEquals
+                    value: "init"
+                  - key: '{{ "{{" }} (element.volumeMounts || `[]`)[?name == ''auth-daemon-config''] | length(@) {{ "}}" }}'
+                    operator: GreaterThan
+                    value: 0
+          - list: "request.object.spec.ephemeralContainers || []"
+            deny:
+              conditions:
+                all:
+                  - key: '{{ "{{" }} (element.volumeMounts || `[]`)[?name == ''auth-daemon-config''] | length(@) {{ "}}" }}'
+                    operator: GreaterThan
+                    value: 0
+    - name: block-exec-into-auth-daemon-pods
+      match:
+        resources:
+          kinds:
+            - Pod/exec
+          namespaceSelector:
+            matchLabels:
+              app.kubernetes.io/managed-by: sessionspaces
+      context:
+        - name: podContainers
+          apiCall:
+            urlPath: '/api/v1/namespaces/{{ "{{" }}request.namespace{{ "}}" }}/pods/{{ "{{" }}request.name{{ "}}" }}'
+            jmesPath: "spec.containers"
+      validate:
+        deny:
+          conditions:
+            any:
+              - key: '{{ "{{" }} podContainers[?contains(image, ''ghcr.io/diamondlightsource/workflows-auth-daemon'')] | length(@) {{ "}}" }}'
+                operator: GreaterThan
+                value: 0

charts/workflows/templates/sessionspace-clusterpolicy.yaml

@@ -142,3 +142,30 @@ spec:
       expression: resource.Get("v1", "secrets", "workflows", "artifact-s3")
   generate:
     - expression: generator.Apply(variables.targetNs, [variables.sourceSecret])
+---
+apiVersion: policies.kyverno.io/v1alpha1
+kind: GeneratingPolicy
+metadata:
+  name: copy-auth-daemon-config
+spec:
+  evaluation:
+    generateExisting:
+      enabled: false
+    synchronize:
+      enabled: false
+  matchConstraints:
+    resourceRules:
+    - apiGroups:   [""]
+      apiVersions: ["v1"]
+      operations:  ["CREATE"]
+      resources:   ["namespaces"]
+    namespaceSelector:
+      matchLabels:
+          app.kubernetes.io/managed-by: sessionspaces
+  variables:
+    - name: targetNs
+      expression: "object.metadata.name"
+    - name: sourceSecret
+      expression: resource.Get("v1", "secrets", "workflows", "auth-daemon-config")
+  generate:
+    - expression: generator.Apply(variables.targetNs, [variables.sourceSecret])

charts/workflows/templates/synchronize-auth-daemon-config.yaml

@@ -0,0 +1,132 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: auth-daemon-config-cloner
+  namespace: workflows
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: auth-daemon-config-cloner
+rules:
+  - apiGroups: [""]
+    resources: ["namespaces"]
+    verbs: ["list", "get", "watch"]
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["create", "update", "patch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: auth-daemon-config-cloner
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: auth-daemon-config-cloner
+subjects:
+  - kind: ServiceAccount
+    name: auth-daemon-config-cloner
+    namespace: workflows
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: auth-daemon-config-cloner
+  namespace: workflows
+rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    resourceNames: ["auth-daemon-config"]
+    verbs: ["get"]
+  - apiGroups: ["batch"]
+    resources: ["cronjobs"]
+    verbs: ["get"]
+  - apiGroups: ["batch"]
+    resources: ["jobs"]
+    verbs: ["create"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: auth-daemon-config-cloner
+  namespace: workflows
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: auth-daemon-config-cloner
+subjects:
+  - kind: ServiceAccount
+    name: auth-daemon-config-cloner
+    namespace: workflows
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: copy-auth-daemon-config
+  namespace: workflows
+spec:
+  # Suspended: intended to be triggered manually on install and after secret rotation.
+  # eg kubectl create job --from=cronjob/copy-auth-daemon-config -nworkflows copy-auth-daemon-config-$(date +%s)
+  suspend: true
+  schedule: "0 3 * * *"
+  successfulJobsHistoryLimit: 1
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      backoffLimit: 0
+      ttlSecondsAfterFinished: 300
+      template:
+        spec:
+          serviceAccountName: auth-daemon-config-cloner
+          restartPolicy: Never
+          containers:
+            - name: sync
+              image: alpine/kubectl:1.34.2
+              imagePullPolicy: IfNotPresent
+              command: ["/bin/sh", "-c"]
+              args:
+                - |
+                  set -euo pipefail
+
+                  SOURCE_NAMESPACE="${SOURCE_NAMESPACE:-workflows}"
+                  SECRET_NAME="${SECRET_NAME:-auth-daemon-config}"
+                  LABEL_SELECTOR="${LABEL_SELECTOR:-app.kubernetes.io/managed-by=sessionspaces}"
+
+                  echo "Starting sync of ${SOURCE_NAMESPACE}/${SECRET_NAME} to namespaces with ${LABEL_SELECTOR}"
+
+                  if ! kubectl -n "${SOURCE_NAMESPACE}" get secret "${SECRET_NAME}" >/dev/null 2>&1; then
+                    echo "ERROR: source secret ${SOURCE_NAMESPACE}/${SECRET_NAME} not found" >&2
+                    exit 1
+                  fi
+
+                  CONFIG_B64="$(kubectl -n "${SOURCE_NAMESPACE}" get secret "${SECRET_NAME}" -o jsonpath='{.data.config\.yaml}' || true)"
+
+                  if [ -z "${CONFIG_B64}" ]; then
+                    echo "ERROR: expected key 'config.yaml' missing from secret" >&2
+                    exit 2
+                  fi
+
+                  for ns in $(kubectl get ns -l "${LABEL_SELECTOR}" -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
+                    echo "Syncing into namespace: ${ns}"
+                    kubectl apply --server-side --force-conflicts -f - <<EOF
+                  apiVersion: v1
+                  kind: Secret
+                  metadata:
+                    name: ${SECRET_NAME}
+                    namespace: ${ns}
+                  type: Opaque
+                  data:
+                    config.yaml: ${CONFIG_B64}
+                  EOF
+                  done
+
+                  echo "Sync completed."
+              env:
+                - name: SOURCE_NAMESPACE
+                  value: "workflows"
+                - name: SECRET_NAME
+                  value: "auth-daemon-config"
+                - name: LABEL_SELECTOR
+                  value: "app.kubernetes.io/managed-by=sessionspaces"