Better support to treat events from HepMC event files (multi-timeframe case)

Some users give existing .hepmc files to the event generator.

So far, this was problematic because separate timeframes will
read from the same file and due to a lack of synchronization, each
timeframe read the same events.

This is now fixed. Whenever an hepmc event file is given
as input to O2DPG-MC, we make sure that timeframes read from the HepMC
file incrementally. This is achieved through:

- sequentially producing events for timeframes
- communication of the number processed from one timeframe to the next
  (via HepMCEventSkip.json file).

(cherry picked from commit 74684c5)

Author: sawenzel

MC/bin/o2dpg_sim_config.py

@@ -108,6 +108,13 @@ def add(cfg, flatconfig):
            d[sk] = flatconfig[entry]
            cfg[mk] = d
 
+    # ----- special setting for hepmc generator -----
+    if args.gen == "hepmc":
+      eventSkipPresent = config.get("HepMC",{}).get("eventsToSkip")
+      if eventSkipPresent == None:
+          # add it
+          add(config, {"HepMC.eventsToSkip" : '${HEPMCEVENTSKIP:-0}'})
+
     # ----- add default settings -----
 
     add(config, {"MFTBase.buildAlignment" : "true"})

MC/bin/o2dpg_sim_workflow.py

@@ -704,15 +704,29 @@ def getDPL_global_options(bigshm=False, ccdbbackend=True):
 
    # (separate) event generation task
    sep_event_mode = args.event_gen_mode == 'separated'
-   SGNGENtask=createTask(name='sgngen_'+str(tf), needs=signalneeds, tf=tf, cwd='tf'+str(tf), lab=["GEN"],
+   sgngenneeds=signalneeds
+   # for HepMC we need some special treatment since we need
+   # to ensure that different timeframes read different events from this file
+   if GENERATOR=="hepmc" and tf > 1:
+      sgngenneeds=signalneeds + ['sgngen_' + str(tf-1)] # we serialize event generation
+   SGNGENtask=createTask(name='sgngen_'+str(tf), needs=sgngenneeds, tf=tf, cwd='tf'+str(tf), lab=["GEN"],
                          cpu=1, mem=1000)
-   SGNGENtask['cmd']='${O2_ROOT}/bin/o2-sim --noGeant -j 1 --field ccdb --vertexMode kCCDB'           \
+
+   SGNGENtask['cmd']=''
+   if GENERATOR=="hepmc" and tf > 1:
+     # determine the skip number
+     cmd = 'export HEPMCEVENTSKIP=$(${O2DPG_ROOT}/UTILS/ReadHepMCEventSkip.sh ../HepMCEventSkip.json ' + str(tf) + ');'
+     SGNGENtask['cmd'] = cmd
+   SGNGENtask['cmd'] +='${O2_ROOT}/bin/o2-sim --noGeant -j 1 --field ccdb --vertexMode kCCDB'         \
                      + ' --run ' + str(args.run) + ' ' + str(CONFKEY) + str(TRIGGER)                  \
                      + ' -g ' + str(GENERATOR) + ' ' + str(INIFILE) + ' -o genevents ' + embeddinto   \
                      + ('', ' --timestamp ' + str(args.timestamp))[args.timestamp!=-1]                \
                      + ' --seed ' + str(TFSEED) + ' -n ' + str(NSIGEVENTS)
+
    if args.pregenCollContext == True:
       SGNGENtask['cmd'] += ' --fromCollContext collisioncontext.root:' + signalprefix
+   if GENERATOR=="hepmc":
+      SGNGENtask['cmd'] += "; RC=$?; ${O2DPG_ROOT}/UTILS/UpdateHepMCEventSkip.sh ../HepMCEventSkip.json " + str(tf) + '; [[ ${RC} == 0 ]]'
    if sep_event_mode == True:
       workflow['stages'].append(SGNGENtask)
       signalneeds = signalneeds + [SGNGENtask['name']]
@@ -731,7 +745,6 @@ def getDPL_global_options(bigshm=False, ccdbbackend=True):
       SGNtask['cmd'] += ' --readoutDetectors ' + " ".join(activeDetectors)
    if args.pregenCollContext == True:
       SGNtask['cmd'] += ' --fromCollContext collisioncontext.root'
-
    workflow['stages'].append(SGNtask)
 
    # some tasks further below still want geometry + grp in fixed names, so we provide it here

UTILS/ReadHepMCEventSkip.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# Path to the JSON file
+JSON_FILE=$1
+tf=$2
+JQCOMMAND="jq '[.[] | select(.tf < ${tf}) | .HepMCEventCount] | add' ${JSON_FILE}"
+eval ${JQCOMMAND}

UTILS/UpdateHepMCEventSkip.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# Path to the JSON file
+JSON_FILE=${1:-HepMC_EventSkip_ALT.json}
+tf=$2
+
+# Value to set or add
+EVENTS=$(grep "DISTRIBUTING" ../tf${tf}/sgngen_*.log | tail -n 1 | awk '//{print $5}')
+
+[ -f $JSON_FILE ] || echo "[]" > ${JSON_FILE} # init json file if it doesn't exist
+# insert event count ... if a count for this tf does not already exist
+JQ_COMMAND="jq 'if any(.[]; .tf == ${tf}) then . else . + [{\"tf\": ${tf}, "HepMCEventCount": ${EVENTS}}] end' ${JSON_FILE} > tmp_123.json; mv tmp_123.json ${JSON_FILE}"
+eval ${JQ_COMMAND}