From 19d1cb16303c6ec82c04666939e31ac453d1052a Mon Sep 17 00:00:00 2001 From: David Igandan Date: Tue, 23 Jun 2026 18:20:24 +0100 Subject: [PATCH 1/7] create new cluster submission template --- testing-resources/templates/dc-sim.yaml | 81 +++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 testing-resources/templates/dc-sim.yaml diff --git a/testing-resources/templates/dc-sim.yaml b/testing-resources/templates/dc-sim.yaml new file mode 100644 index 0000000..a44e342 --- /dev/null +++ b/testing-resources/templates/dc-sim.yaml @@ -0,0 +1,81 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ClusterWorkflowTemplate +metadata: + name: dc-sim + labels: + workflows.diamond.ac.uk/science-group-mx: "true" + annotations: + workflows.argoproj.io/title: Simulated Data Collection + workflows.argoproj.io/description: | + Runs a simulated data collection (dc_sim) using existing data, via the + dlstbx.wrap wrapper. The job parameters travel inside the serialised + recipewrapper file; this template only needs its path. The wrapper + reports back to Zocalo (queue reduce.dc_sim) only on success. + workflows.diamond.ac.uk/repository: https://github.com/DiamondLightSource/mx-workflows + +spec: + entrypoint: workflow-entry + arguments: + parameters: + # Path to the serialised recipewrapper file written by the + # DLS Workflows Cluster service before submission. Must be on a + # filesystem visible to this pod (see the dls volume mount below). + - name: recipewrapper + value: "" + + # Mount the DLS shared filesystem so the pod can read the recipewrapper + # file and any source data referenced by the dc_sim job parameters. + volumes: + - name: dls + hostPath: + path: /dls + type: Directory + # Zocalo configuration + RabbitMQ credentials, needed for the wrapper to + # connect to the transport and send the dc_sim completion message. + - name: zocalo-secrets + secret: + secretName: zocalo-rabbitmq-credentials + + templates: + - name: run-dc-sim + inputs: + parameters: + - name: recipewrapper + script: + image: gitlab.diamond.ac.uk:5050/scisoft/mx/python-dlstbx:044a56cd + volumeMounts: + - name: dls + mountPath: /dls + - name: zocalo-secrets + mountPath: /etc/zocalo + readOnly: true + env: + # Point dlstbx.wrap at the Zocalo configuration file. + - name: ZOCALO_CONFIG + value: /etc/zocalo/configuration.yaml + resources: + requests: + cpu: "1" + memory: 2Gi + limits: + cpu: "2" + memory: 4Gi + command: [bash] + source: | + set -euo pipefail + echo "Running dc_sim wrapper with recipewrapper {{inputs.parameters.recipewrapper}}" + dlstbx.wrap \ + --wrap dc_sim \ + --recipewrapper "{{inputs.parameters.recipewrapper}}" + + - name: workflow-entry + # Equivalent of the old Slurm time_limit "1:00:00". + activeDeadlineSeconds: 3600 + dag: + tasks: + - name: run-dc-sim + template: run-dc-sim + arguments: + parameters: + - name: recipewrapper + value: "{{workflow.parameters.recipewrapper}}" From b0f06cc3ae55fa8626381a9a7d11c810608c13cd Mon Sep 17 00:00:00 2001 From: David Igandan Date: Tue, 23 Jun 2026 18:24:58 +0100 Subject: [PATCH 2/7] test --- .workflows-lint.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.workflows-lint.yaml b/.workflows-lint.yaml index 42571b1..e96a5d3 100644 --- a/.workflows-lint.yaml +++ b/.workflows-lint.yaml @@ -1,2 +1,3 @@ charts: - testing-resources + - dc-sim From 012ce24df955b2a8424ddd06767943d39dc22e69 Mon Sep 17 00:00:00 2001 From: David Igandan Date: Tue, 23 Jun 2026 18:38:27 +0100 Subject: [PATCH 3/7] works --- wss/templates/dc-sim.yaml | 79 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 wss/templates/dc-sim.yaml diff --git a/wss/templates/dc-sim.yaml b/wss/templates/dc-sim.yaml new file mode 100644 index 0000000..3658068 --- /dev/null +++ b/wss/templates/dc-sim.yaml @@ -0,0 +1,79 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ClusterWorkflowTemplate +metadata: + name: dc-sim + labels: + workflows.diamond.ac.uk/science-group-mx: "true" + annotations: + workflows.argoproj.io/title: Simulated Data Collection + workflows.argoproj.io/description: | + Runs a simulated data collection (dc_sim) using existing data, via the + dlstbx.wrap wrapper. The job parameters travel inside the serialised + recipewrapper file; this template only needs its path. The wrapper + reports back to Zocalo (queue reduce.dc_sim) only on success. + workflows.diamond.ac.uk/repository: https://github.com/DiamondLightSource/mx-workflows + +spec: + entrypoint: workflow-entry + arguments: + parameters: + # Path to the serialised recipewrapper file written by the + # DLS Workflows Cluster service before submission. Must be on a + # filesystem visible to this pod (see the dls volume mount below). + - name: recipewrapper + value: "" + + # Mount the DLS shared filesystem so the pod can read the recipewrapper + # file and any source data referenced by the dc_sim job parameters. + volumes: + - name: dls + hostPath: + path: /dls + type: Directory + # Zocalo configuration + RabbitMQ credentials, needed for the wrapper to + # connect to the transport and send the dc_sim completion message. + - name: zocalo-secrets + secret: + secretName: zocalo-rabbitmq-credentials + + templates: + - name: run-dc-sim + inputs: + parameters: + - name: recipewrapper + script: + image: gitlab.diamond.ac.uk:5050/scisoft/mx/python-dlstbx:044a56cd + volumeMounts: + - name: dls + mountPath: /dls + - name: zocalo-secrets + mountPath: /etc/zocalo + readOnly: true + env: + # Point dlstbx.wrap at the Zocalo configuration file. + - name: ZOCALO_CONFIG + value: /etc/zocalo/configuration.yaml + resources: + requests: + cpu: "1" + memory: 2Gi + limits: + cpu: "2" + memory: 4Gi + command: [bash] + source: | + set -euo pipefail + echo "Running dc_sim wrapper with recipewrapper {{inputs.parameters.recipewrapper}}" + dlstbx.wrap \ + --wrap dc_sim \ + --recipewrapper "{{inputs.parameters.recipewrapper}}" + + - name: workflow-entry + dag: + tasks: + - name: run-dc-sim + template: run-dc-sim + arguments: + parameters: + - name: recipewrapper + value: "{{workflow.parameters.recipewrapper}}" From b9e5b8ff5e8cec3726e2d2b5cbf1ea3390320a1e Mon Sep 17 00:00:00 2001 From: David Igandan Date: Tue, 23 Jun 2026 18:45:36 +0100 Subject: [PATCH 4/7] works2 --- wss/templates/dc-sim.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/wss/templates/dc-sim.yaml b/wss/templates/dc-sim.yaml index 3658068..bf11251 100644 --- a/wss/templates/dc-sim.yaml +++ b/wss/templates/dc-sim.yaml @@ -63,10 +63,10 @@ spec: command: [bash] source: | set -euo pipefail - echo "Running dc_sim wrapper with recipewrapper {{inputs.parameters.recipewrapper}}" + echo "Running dc_sim wrapper with recipewrapper {{`{{inputs.parameters.recipewrapper}}`}}" dlstbx.wrap \ --wrap dc_sim \ - --recipewrapper "{{inputs.parameters.recipewrapper}}" + --recipewrapper "{{`{{inputs.parameters.recipewrapper}}`}}" - name: workflow-entry dag: @@ -76,4 +76,4 @@ spec: arguments: parameters: - name: recipewrapper - value: "{{workflow.parameters.recipewrapper}}" + value: "{{`{{workflow.parameters.recipewrapper}}`}}" From b94a77fcb41a31bd17a11255762483b437d49160 Mon Sep 17 00:00:00 2001 From: David Igandan Date: Tue, 23 Jun 2026 18:52:55 +0100 Subject: [PATCH 5/7] Move dc-sim into its own wss chart; fix lint config path --- .workflows-lint.yaml | 2 +- wss/Chart.yaml | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 wss/Chart.yaml diff --git a/.workflows-lint.yaml b/.workflows-lint.yaml index e96a5d3..7ea3213 100644 --- a/.workflows-lint.yaml +++ b/.workflows-lint.yaml @@ -1,3 +1,3 @@ charts: - testing-resources - - dc-sim + - wss diff --git a/wss/Chart.yaml b/wss/Chart.yaml new file mode 100644 index 0000000..3d45a40 --- /dev/null +++ b/wss/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: dc-sim +description: run dc-sim +type: application + +version: 0.1.0 From a035b6bd247192d808f891d342f68dc2e5eaa8e3 Mon Sep 17 00:00:00 2001 From: David Igandan Date: Tue, 30 Jun 2026 16:09:45 +0100 Subject: [PATCH 6/7] dc-sim test template --- .workflows-lint.yaml | 2 +- testing-resources/templates/dc-sim.yaml | 81 ----------------------- wss/templates/dc-sim.yaml | 85 ++++++------------------- 3 files changed, 22 insertions(+), 146 deletions(-) delete mode 100644 testing-resources/templates/dc-sim.yaml diff --git a/.workflows-lint.yaml b/.workflows-lint.yaml index 7ea3213..e96a5d3 100644 --- a/.workflows-lint.yaml +++ b/.workflows-lint.yaml @@ -1,3 +1,3 @@ charts: - testing-resources - - wss + - dc-sim diff --git a/testing-resources/templates/dc-sim.yaml b/testing-resources/templates/dc-sim.yaml deleted file mode 100644 index a44e342..0000000 --- a/testing-resources/templates/dc-sim.yaml +++ /dev/null @@ -1,81 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: ClusterWorkflowTemplate -metadata: - name: dc-sim - labels: - workflows.diamond.ac.uk/science-group-mx: "true" - annotations: - workflows.argoproj.io/title: Simulated Data Collection - workflows.argoproj.io/description: | - Runs a simulated data collection (dc_sim) using existing data, via the - dlstbx.wrap wrapper. The job parameters travel inside the serialised - recipewrapper file; this template only needs its path. The wrapper - reports back to Zocalo (queue reduce.dc_sim) only on success. - workflows.diamond.ac.uk/repository: https://github.com/DiamondLightSource/mx-workflows - -spec: - entrypoint: workflow-entry - arguments: - parameters: - # Path to the serialised recipewrapper file written by the - # DLS Workflows Cluster service before submission. Must be on a - # filesystem visible to this pod (see the dls volume mount below). - - name: recipewrapper - value: "" - - # Mount the DLS shared filesystem so the pod can read the recipewrapper - # file and any source data referenced by the dc_sim job parameters. - volumes: - - name: dls - hostPath: - path: /dls - type: Directory - # Zocalo configuration + RabbitMQ credentials, needed for the wrapper to - # connect to the transport and send the dc_sim completion message. - - name: zocalo-secrets - secret: - secretName: zocalo-rabbitmq-credentials - - templates: - - name: run-dc-sim - inputs: - parameters: - - name: recipewrapper - script: - image: gitlab.diamond.ac.uk:5050/scisoft/mx/python-dlstbx:044a56cd - volumeMounts: - - name: dls - mountPath: /dls - - name: zocalo-secrets - mountPath: /etc/zocalo - readOnly: true - env: - # Point dlstbx.wrap at the Zocalo configuration file. - - name: ZOCALO_CONFIG - value: /etc/zocalo/configuration.yaml - resources: - requests: - cpu: "1" - memory: 2Gi - limits: - cpu: "2" - memory: 4Gi - command: [bash] - source: | - set -euo pipefail - echo "Running dc_sim wrapper with recipewrapper {{inputs.parameters.recipewrapper}}" - dlstbx.wrap \ - --wrap dc_sim \ - --recipewrapper "{{inputs.parameters.recipewrapper}}" - - - name: workflow-entry - # Equivalent of the old Slurm time_limit "1:00:00". - activeDeadlineSeconds: 3600 - dag: - tasks: - - name: run-dc-sim - template: run-dc-sim - arguments: - parameters: - - name: recipewrapper - value: "{{workflow.parameters.recipewrapper}}" diff --git a/wss/templates/dc-sim.yaml b/wss/templates/dc-sim.yaml index bf11251..dcec79c 100644 --- a/wss/templates/dc-sim.yaml +++ b/wss/templates/dc-sim.yaml @@ -5,75 +5,32 @@ metadata: labels: workflows.diamond.ac.uk/science-group-mx: "true" annotations: - workflows.argoproj.io/title: Simulated Data Collection - workflows.argoproj.io/description: | - Runs a simulated data collection (dc_sim) using existing data, via the - dlstbx.wrap wrapper. The job parameters travel inside the serialised - recipewrapper file; this template only needs its path. The wrapper - reports back to Zocalo (queue reduce.dc_sim) only on success. - workflows.diamond.ac.uk/repository: https://github.com/DiamondLightSource/mx-workflows - + workflows.diamond.ac.uk/repository: "https://github.com/DiamondLightSource/mx-workflows" spec: entrypoint: workflow-entry arguments: parameters: - # Path to the serialised recipewrapper file written by the - # DLS Workflows Cluster service before submission. Must be on a - # filesystem visible to this pod (see the dls volume mount below). - - name: recipewrapper + - name: RECIPEWRAP value: "" - # Mount the DLS shared filesystem so the pod can read the recipewrapper - # file and any source data referenced by the dc_sim job parameters. - volumes: - - name: dls - hostPath: - path: /dls - type: Directory - # Zocalo configuration + RabbitMQ credentials, needed for the wrapper to - # connect to the transport and send the dc_sim completion message. - - name: zocalo-secrets - secret: - secretName: zocalo-rabbitmq-credentials - templates: - - name: run-dc-sim - inputs: - parameters: - - name: recipewrapper - script: - image: gitlab.diamond.ac.uk:5050/scisoft/mx/python-dlstbx:044a56cd - volumeMounts: - - name: dls - mountPath: /dls - - name: zocalo-secrets - mountPath: /etc/zocalo - readOnly: true - env: - # Point dlstbx.wrap at the Zocalo configuration file. - - name: ZOCALO_CONFIG - value: /etc/zocalo/configuration.yaml - resources: - requests: - cpu: "1" - memory: 2Gi - limits: - cpu: "2" - memory: 4Gi - command: [bash] - source: | - set -euo pipefail - echo "Running dc_sim wrapper with recipewrapper {{`{{inputs.parameters.recipewrapper}}`}}" - dlstbx.wrap \ - --wrap dc_sim \ - --recipewrapper "{{`{{inputs.parameters.recipewrapper}}`}}" + - name: run-dc-sim + inputs: + parameters: + - name: RECIPEWRAP + script: + image: ghcr.io/diamondlightsource/dlstbx + command: [bash] + source: | + echo dlstbx.wrap --wrap dc_sim --recipewrapper "{{`{{inputs.parameters.RECIPEWRAP}}`}}" >runinfo + dlstbx.wrap --wrap dc_sim --recipewrapper "{{`{{inputs.parameters.RECIPEWRAP}}`}}" - - name: workflow-entry - dag: - tasks: - - name: run-dc-sim - template: run-dc-sim - arguments: - parameters: - - name: recipewrapper - value: "{{`{{workflow.parameters.recipewrapper}}`}}" + - name: workflow-entry + dag: + tasks: + - name: run-dc-sim + template: run-dc-sim + arguments: + parameters: + - name: RECIPEWRAP + value: "{{`{{workflow.parameters.RECIPEWRAP}}`}}" From b438e688aeb38ce0ee9f2d9750b8f6c4421e37cb Mon Sep 17 00:00:00 2001 From: David Igandan Date: Wed, 1 Jul 2026 16:30:46 +0100 Subject: [PATCH 7/7] change name win workflows-lint.yaml to wss --- .workflows-lint.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.workflows-lint.yaml b/.workflows-lint.yaml index e96a5d3..7ea3213 100644 --- a/.workflows-lint.yaml +++ b/.workflows-lint.yaml @@ -1,3 +1,3 @@ charts: - testing-resources - - dc-sim + - wss