diff --git a/.github/workflows/_kyverno_policy.yaml b/.github/workflows/_kyverno_policy.yaml index 62d6fb155..e60be3371 100644 --- a/.github/workflows/_kyverno_policy.yaml +++ b/.github/workflows/_kyverno_policy.yaml @@ -51,7 +51,7 @@ jobs: - name: Install mpi-operator CRDs run: | - kubectl apply --server-side -f https://raw.githubusercontent.com/kubeflow/mpi-operator/v0.7.0/deploy/v2beta1/mpi-operator.yaml + curl -sL https://raw.githubusercontent.com/kubeflow/mpi-operator/v0.7.0/deploy/v2beta1/mpi-operator.yaml | yq e 'select(.kind == "CustomResourceDefinition")' | kubectl apply --server-side -f - - name: Deploy and wait for Policies, ClusterRoles, and ClusterRoleBindings run: | diff --git a/charts/sessionspaces/Chart.yaml b/charts/sessionspaces/Chart.yaml index 159b94b60..27d222e54 100644 --- a/charts/sessionspaces/Chart.yaml +++ b/charts/sessionspaces/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: sessionspaces description: Namespace controller for creating session namespaces type: application -version: 0.3.28 +version: 0.3.29 appVersion: 0.1.6 dependencies: - name: common diff --git a/charts/sessionspaces/test-policy/pod-securitycontext/chainsaw-test.yaml b/charts/sessionspaces/test-policy/pod-securitycontext/chainsaw-test.yaml index 912594695..5aecac2cf 100644 --- a/charts/sessionspaces/test-policy/pod-securitycontext/chainsaw-test.yaml +++ b/charts/sessionspaces/test-policy/pod-securitycontext/chainsaw-test.yaml @@ -3,6 +3,13 @@ kind: Test metadata: name: pod-securitycontext spec: + timeouts: + apply: 45s + assert: 30s + cleanup: 2m + delete: 1m + error: 30s + exec: 15s namespaceTemplate: metadata: labels: diff --git a/charts/workflows-cluster/Chart.yaml b/charts/workflows-cluster/Chart.yaml index f0de247de..871874a16 100644 --- a/charts/workflows-cluster/Chart.yaml +++ b/charts/workflows-cluster/Chart.yaml @@ -3,7 +3,7 @@ name: workflows-cluster description: A virtual cluster for Data Analysis workflows type: application -version: 0.13.7 +version: 0.13.8 dependencies: - name: common version: 2.23.0 diff --git a/charts/workflows-cluster/charts/secrets/templates/auth-daemon.yaml b/charts/workflows-cluster/charts/secrets/templates/auth-daemon.yaml new file mode 100644 index 000000000..1ce1dcf02 --- /dev/null +++ b/charts/workflows-cluster/charts/secrets/templates/auth-daemon.yaml @@ -0,0 +1,16 @@ +{{- if eq .Values.cluster "argus" }} +{{- else if eq .Values.cluster "pollux" }} +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + name: auth-daemon-config + namespace: workflows +spec: + encryptedData: + config.yaml: AgCh9l99Vb3PxDSCwApnc5mJO2HnI9W+Y3ENS7y1nTXvzeAnBuasjQmJO8tRgC2Es/BBhqYr6/pGCiU7KTTx5wNxoue4gMc6O/LvJhoc2KjLvdadXSuGPyTIXcBSDuFl4ErrbJXX3fXmSVAFTJtLkdcoMvaMq2zeqOp/dNWAE6slvjDF49BoLfy8LOo1MwIz/8r6DLD6Qpy5SUjlvoHL6ZBVNPlM3jXIcaooHaWzdq+tC/tBg6Za+4sjEzownWwl9pQBx1Ru/4kYnfL5aNQ/olQLKNnNdC3c6i0dSNB7VZHMqTmQIsRdA2WU/QSPBoVKQLeIBjcw1HNRyMxvuG9ojZ/NsgdXrvd6Nrn4t6nYB1rfUV17syhTtsD9iBK/LVkUPr+KMHX9LcE7B66cfhKrHCYM747bM2cI4Qz6ADG2vZ8enMFfCvq2RcnAYqf9pASScwLsCDNRS72ub0t6/j/8TwxSAy1D1LKaZF+1IGsOzFAhIApA5Lu5pYRqg8qwSToEvWcOIodrIkNDpcJuJWChRiV6Y2JJggKvWOSIBLEWxLd1Od7NjpZcRSfmANhHyfctGBXzu++IcYCcauiGm4vIPZjjvsS1fhTKi7rJ1UOiFA3H/jKuew1ZDn5/CGut48hSqvMQWR7ZYgzUnTycUGCs0eX5XJJ/e778dYIKi2pEfArQnXfNWVSgaT43F/4YJAzPodcE/Md9Nncf/has1BFMafliVDbTVEi9Bb66Uhrlinn1RAaMtNzQ1t4ZGGe6EgtuwgycbyKuKCS1rkApamhh8/+N3bi3NJFCfZn5VTyt0r1qRRfU6Ldapn9+deLE5KQRY64Y/nhW96mdimVY78/U2zdSGtbgADox5HQVIWblD/MzpaYytswW5Ik01JulC1mSP5z6GpDhNZG8q82yLNNtI3glPhHKaLybeOzqOq/lCoDxkQ+S3G0q2l22+wgwJcrv5bM08Q8Ug0uFQZDZT41FctHKxtI4s51mvN0tlS5CmG818HBxG4FmoK4hNV7Qal0ioesUGYNFspz1pLrjl3MNv/wKDvwUWwvbkMG9GYe4doL97QLsWANd5YJNTgA+lqCH3i5vjQRgJuS9uOwgCmy1XjNnujIiiHVY6jzpogjL1B2LXG+ebdYdWlmOr5y+5IOQkkVhpLmrfych/GV3A/+aJJAbogbproUJtnm4ySPWKj4zWULm2h3DuF66DgNp1tLmX0ARduKHGgTD1ixgpLGPxdjIkPew+n6tVwSlyRik42B3lBY3aIdiTtIwEuxp1GPXlC6KHtNPBR45MeVYMUVjJsVtB9dLswZ5BUaozvSOsap85YEKO1foSMJliW1KoMYg20UFbLmA0ZrnDDaF8mwOSen6qCKmSrj4eG/rfFDgYe8Wa7N1AIL5IYSS1jgfbDyCt18opMTeNGr7nTyfqzPQSRaEAC9yJ1Vj0QhJ59ZfUQmAGOLKFlEd2HPmxVpIr8Dz/hJJQC30yT2Wf0ekHg== + template: + metadata: + name: auth-daemon-config + namespace: workflows +{{- else }} +{{- end }} diff --git a/charts/workflows-cluster/staging-values.yaml b/charts/workflows-cluster/staging-values.yaml index 49a51f14f..2a04ab1de 100644 --- a/charts/workflows-cluster/staging-values.yaml +++ b/charts/workflows-cluster/staging-values.yaml @@ -83,6 +83,7 @@ vcluster: "/artifact-s3-secret": "graph-proxy/artifact-s3-secret" "/s3-artifact": "workflows/artifact-s3" "/oidc-bff-config": "workflows/oidc-bff-config" + "/auth-daemon-config": "workflows/auth-daemon-config" "/postgres-passwords": "workflows/postgres-passwords" "/postgres-argo-workflows-password": "workflows/postgres-argo-workflows-password" "/graph-proxy-k6-auth": "k6/graph-proxy-k6-auth" diff --git a/charts/workflows/Chart.yaml b/charts/workflows/Chart.yaml index 9b0541b71..fb85be88d 100644 --- a/charts/workflows/Chart.yaml +++ b/charts/workflows/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: workflows description: Data Analysis workflow orchestration type: application -version: 0.13.47 +version: 0.13.48 dependencies: - name: argo-workflows repository: https://argoproj.github.io/argo-helm diff --git a/charts/workflows/templates/auth-daemon-config-clusterpolicy.yaml b/charts/workflows/templates/auth-daemon-config-clusterpolicy.yaml new file mode 100644 index 000000000..0e9ceb041 --- /dev/null +++ b/charts/workflows/templates/auth-daemon-config-clusterpolicy.yaml @@ -0,0 +1,128 @@ +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: {{ .Release.Name }}-auth-daemon-config +spec: + validationFailureAction: Enforce + background: false + rules: + - name: prohibit-auth-daemon-config-volume + match: + resources: + kinds: + - Pod + namespaceSelector: + matchLabels: + app.kubernetes.io/managed-by: sessionspaces + validate: + message: "auth-daemon-config can only be used by the auth-daemon container" + pattern: + spec: + =(volumes): + - (name): "!auth-daemon-config" + =(secret): + secretName: "!auth-daemon-config" + =(projected): + =(sources): + - =(secret): + name: "!auth-daemon-config" + - name: prohibit-auth-daemon-config-env + match: + resources: + kinds: + - Pod + namespaceSelector: + matchLabels: + app.kubernetes.io/managed-by: sessionspaces + validate: + message: "auth-daemon-config can only be used by the auth-daemon container" + pattern: + spec: + containers: + - (image): "!ghcr.io/diamondlightsource/workflows-auth-daemon:*" + =(env): + - =(valueFrom): + =(secretKeyRef): + name: "!auth-daemon-config" + =(envFrom): + - =(secretRef): + name: "!auth-daemon-config" + =(initContainers): + - (name): "!init" + =(env): + - =(valueFrom): + =(secretKeyRef): + name: "!auth-daemon-config" + =(envFrom): + - =(secretRef): + name: "!auth-daemon-config" + =(ephemeralContainers): + - name: "*" + =(env): + - =(valueFrom): + =(secretKeyRef): + name: "!auth-daemon-config" + =(envFrom): + - =(secretRef): + name: "!auth-daemon-config" + - name: prohibit-auth-daemon-config-volumemounts + match: + resources: + kinds: + - Pod + namespaceSelector: + matchLabels: + app.kubernetes.io/managed-by: sessionspaces + validate: + message: "auth-daemon-config can only be used by the auth-daemon container" + foreach: + - list: "request.object.spec.containers" + deny: + conditions: + all: + - key: '{{ "{{" }} element.name {{ "}}" }}' + operator: NotEquals + value: "wait" + - key: '{{ "{{" }} contains(element.image, ''ghcr.io/diamondlightsource/workflows-auth-daemon'') {{ "}}" }}' + operator: Equals + value: false + - key: '{{ "{{" }} (element.volumeMounts || `[]`)[?name == ''auth-daemon-config''] | length(@) {{ "}}" }}' + operator: GreaterThan + value: 0 + - list: "request.object.spec.initContainers || []" + deny: + conditions: + all: + - key: '{{ "{{" }} element.name {{ "}}" }}' + operator: NotEquals + value: "init" + - key: '{{ "{{" }} (element.volumeMounts || `[]`)[?name == ''auth-daemon-config''] | length(@) {{ "}}" }}' + operator: GreaterThan + value: 0 + - list: "request.object.spec.ephemeralContainers || []" + deny: + conditions: + all: + - key: '{{ "{{" }} (element.volumeMounts || `[]`)[?name == ''auth-daemon-config''] | length(@) {{ "}}" }}' + operator: GreaterThan + value: 0 + - name: block-exec-into-auth-daemon-pods + match: + resources: + kinds: + - Pod/exec + namespaceSelector: + matchLabels: + app.kubernetes.io/managed-by: sessionspaces + context: + - name: podContainers + apiCall: + urlPath: '/api/v1/namespaces/{{ "{{" }}request.namespace{{ "}}" }}/pods/{{ "{{" }}request.name{{ "}}" }}' + jmesPath: "spec.containers" + validate: + deny: + conditions: + any: + - key: '{{ "{{" }} podContainers[?contains(image, ''ghcr.io/diamondlightsource/workflows-auth-daemon'')] | length(@) {{ "}}" }}' + operator: GreaterThan + value: 0 diff --git a/charts/workflows/templates/sessionspace-clusterpolicy.yaml b/charts/workflows/templates/sessionspace-clusterpolicy.yaml index 826e95cb8..c3317fecf 100644 --- a/charts/workflows/templates/sessionspace-clusterpolicy.yaml +++ b/charts/workflows/templates/sessionspace-clusterpolicy.yaml @@ -142,3 +142,30 @@ spec: expression: resource.Get("v1", "secrets", "workflows", "artifact-s3") generate: - expression: generator.Apply(variables.targetNs, [variables.sourceSecret]) +--- +apiVersion: policies.kyverno.io/v1alpha1 +kind: GeneratingPolicy +metadata: + name: copy-auth-daemon-config +spec: + evaluation: + generateExisting: + enabled: false + synchronize: + enabled: false + matchConstraints: + resourceRules: + - apiGroups: [""] + apiVersions: ["v1"] + operations: ["CREATE"] + resources: ["namespaces"] + namespaceSelector: + matchLabels: + app.kubernetes.io/managed-by: sessionspaces + variables: + - name: targetNs + expression: "object.metadata.name" + - name: sourceSecret + expression: resource.Get("v1", "secrets", "workflows", "auth-daemon-config") + generate: + - expression: generator.Apply(variables.targetNs, [variables.sourceSecret]) diff --git a/charts/workflows/templates/synchronize-auth-daemon-config.yaml b/charts/workflows/templates/synchronize-auth-daemon-config.yaml new file mode 100644 index 000000000..dac783c3f --- /dev/null +++ b/charts/workflows/templates/synchronize-auth-daemon-config.yaml @@ -0,0 +1,132 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: auth-daemon-config-cloner + namespace: workflows +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: auth-daemon-config-cloner +rules: + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["list", "get", "watch"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["create", "update", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: auth-daemon-config-cloner +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: auth-daemon-config-cloner +subjects: + - kind: ServiceAccount + name: auth-daemon-config-cloner + namespace: workflows +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: auth-daemon-config-cloner + namespace: workflows +rules: + - apiGroups: [""] + resources: ["secrets"] + resourceNames: ["auth-daemon-config"] + verbs: ["get"] + - apiGroups: ["batch"] + resources: ["cronjobs"] + verbs: ["get"] + - apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["create"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: auth-daemon-config-cloner + namespace: workflows +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: auth-daemon-config-cloner +subjects: + - kind: ServiceAccount + name: auth-daemon-config-cloner + namespace: workflows +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: copy-auth-daemon-config + namespace: workflows +spec: + # Suspended: intended to be triggered manually on install and after secret rotation. + # eg kubectl create job --from=cronjob/copy-auth-daemon-config -nworkflows copy-auth-daemon-config-$(date +%s) + suspend: true + schedule: "0 3 * * *" + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + backoffLimit: 0 + ttlSecondsAfterFinished: 300 + template: + spec: + serviceAccountName: auth-daemon-config-cloner + restartPolicy: Never + containers: + - name: sync + image: alpine/kubectl:1.34.2 + imagePullPolicy: IfNotPresent + command: ["/bin/sh", "-c"] + args: + - | + set -euo pipefail + + SOURCE_NAMESPACE="${SOURCE_NAMESPACE:-workflows}" + SECRET_NAME="${SECRET_NAME:-auth-daemon-config}" + LABEL_SELECTOR="${LABEL_SELECTOR:-app.kubernetes.io/managed-by=sessionspaces}" + + echo "Starting sync of ${SOURCE_NAMESPACE}/${SECRET_NAME} to namespaces with ${LABEL_SELECTOR}" + + if ! kubectl -n "${SOURCE_NAMESPACE}" get secret "${SECRET_NAME}" >/dev/null 2>&1; then + echo "ERROR: source secret ${SOURCE_NAMESPACE}/${SECRET_NAME} not found" >&2 + exit 1 + fi + + CONFIG_B64="$(kubectl -n "${SOURCE_NAMESPACE}" get secret "${SECRET_NAME}" -o jsonpath='{.data.config\.yaml}' || true)" + + if [ -z "${CONFIG_B64}" ]; then + echo "ERROR: expected key 'config.yaml' missing from secret" >&2 + exit 2 + fi + + for ns in $(kubectl get ns -l "${LABEL_SELECTOR}" -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do + echo "Syncing into namespace: ${ns}" + kubectl apply --server-side --force-conflicts -f - <