Merge branch 'AliceO2Group:master' into master

Author: shreyasiacharya

DATA/common/setenv_calib.sh

@@ -34,7 +34,7 @@ if [[ $SYNCMODE == 1 ]] && has_processing_step ENTROPY_ENCODER && [[ ! -z "$WORK
 # for async recalibration
 if has_detector_calib EMC && has_detector_reco EMC && [[ $SYNCMODE != 1 ]]; then CAN_DO_CALIB_EMC_ASYNC_RECALIB=1; else CAN_DO_CALIB_EMC_ASYNC_RECALIB=0; fi
 if [[ $SYNCMODE != 1 ]] && has_detector_reco TPC; then CAN_DO_CALIB_ASYNC_EXTRACTTPCCURRENTS=1; else CAN_DO_CALIB_ASYNC_EXTRACTTPCCURRENTS=0; fi
-if [[ $SYNCMODE != 1 ]] && has_detector_reco TPC && has_detector_reco ITS && has_detector_reco FT0; then CAN_DO_CALIB_ASYNC_EXTRACTTIMESERIES=1; else CAN_DO_CALIB_ASYNC_EXTRACTTIMESERIES=0; fi
+if [[ $SYNCMODE != 1 ]] && has_detector_reco TPC; then CAN_DO_CALIB_ASYNC_EXTRACTTIMESERIES=1; else CAN_DO_CALIB_ASYNC_EXTRACTTIMESERIES=0; fi
 
 # additional individual settings for calibration workflows
 if has_detector CTP; then export CALIB_TPC_SCDCALIB_CTP_INPUT="--enable-ctp"; else export CALIB_TPC_SCDCALIB_CTP_INPUT=""; fi

DATA/production/calib/tpc-laser-filter.sh

@@ -4,7 +4,7 @@ source common/setenv.sh
 
 source common/getCommonArgs.sh
 
-source common/gen_topo_helper_functions.sh 
+source common/gen_topo_helper_functions.sh
 
 FILEWORKDIR="/home/wiechula/processData/inputFilesTracking/triggeredLaser"
 
@@ -63,7 +63,7 @@ REMAP="--condition-remap \"file://${FILEWORKDIR}=GLO/Config/GRPECS,GLO/Config/GR
 RECO_CONFIG="NameConf.mDirGRP=$FILEWORKDIR;"
 RECO_CONFIG+="NameConf.mDirGeom=$FILEWORKDIR2;"
 RECO_CONFIG+="NameConf.mDirCollContext=$FILEWORKDIR;"
-RECO_CONFIG+="NameConf.mDirMatLUT=$FILEWORKDIR;"   
+RECO_CONFIG+="NameConf.mDirMatLUT=$FILEWORKDIR;"
 RECO_CONFIG+="align-geom.mDetectors=none;"
 RECO_CONFIG+="GPU_global.deviceType=$GPUTYPE;"
 RECO_CONFIG+="GPU_proc.tpcIncreasedMinClustersPerRow=500000;"
@@ -81,7 +81,8 @@ RECO_CONFIG+="GPU_rec_tpc.trackFollowingMaxRowGap=15;GPU_rec_tpc.trackFollowingM
 WORKFLOW=
 add_W o2-dpl-raw-proxy "--dataspec \"$PROXY_INSPEC\" --inject-missing-data --channel-config \"name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1\"" "" 0
 add_W o2-tpc-raw-to-digits-workflow "--ignore-grp --input-spec \"$CALIB_INSPEC\" --remove-duplicates --pipeline tpc-raw-to-digits-0:20 --send-ce-digits " "${RAWDIGIT_CONFIG}"
-add_W o2-tpc-reco-workflow " ${TPC_CORR_SCALING:-} --disable-ctp-lumi-request --input-type digitizer --output-type \"tracks,disable-writer,clusters\" --disable-mc --pipeline tpc-zsEncoder:20,tpc-tracker:8 ${GPU_CONFIG} ${REMAP} " "${RECO_CONFIG}"
+add_W o2-tpc-scaler-workflow "--disable-IDC-scalers --disable-ctp-lumi-request"
+add_W o2-tpc-reco-workflow " --input-type digitizer --output-type \"tracks,disable-writer,clusters\" --disable-mc --pipeline tpc-zsEncoder:20,tpc-tracker:8 ${GPU_CONFIG} ${REMAP} " "${RECO_CONFIG}"
 add_W o2-tpc-laser-track-filter "" "" 0
 add_W o2-dpl-output-proxy " --proxy-name tpc-laser-input-proxy --proxy-channel-name tpc-laser-input-proxy --dataspec \"$PROXY_OUTSPEC\" --channel-config \"name=tpc-laser-input-proxy,method=connect,type=push,transport=zeromq,rateLogging=0\" " "" 0
 add_QC_from_apricot "${QC_CONFIG}" "--local --host localhost"

DATA/production/calib/tpc-laser.sh

@@ -4,7 +4,7 @@ source common/setenv.sh
 
 source common/getCommonArgs.sh
 
-source common/gen_topo_helper_functions.sh 
+source common/gen_topo_helper_functions.sh
 
 FILEWORKDIR="/home/wiechula/processData/inputFilesTracking/triggeredLaser"
 
@@ -77,7 +77,7 @@ if [[ ! -z ${TPC_CALIB_LANES_PAD_RAW:-} ]]; then
     num_lanes=${TPC_CALIB_LANES_PAD_RAW}
 fi
 
-EXTRA_CONFIG="--calib-type ce --publish-after-tfs ${publish_after} --max-events ${max_events} --lanes ${num_lanes} --check-calib-infos" 
+EXTRA_CONFIG="--calib-type ce --publish-after-tfs ${publish_after} --max-events ${max_events} --lanes ${num_lanes} --check-calib-infos"
 
 LASER_DECODER_ADD=''
 
@@ -100,7 +100,8 @@ RECO_CONFIG+="GPU_rec_tpc.trackFollowingMaxRowGap=15;GPU_rec_tpc.trackFollowingM
 WORKFLOW=
 add_W o2-dpl-raw-proxy "--dataspec \"$PROXY_INSPEC\" --inject-missing-data --channel-config \"name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1\"" "" 0
 add_W o2-tpc-raw-to-digits-workflow "--ignore-grp --input-spec \"$CALIB_INSPEC\" --remove-duplicates --pipeline tpc-raw-to-digits-0:20 --send-ce-digits " "${RAWDIGIT_CONFIG}"
-add_W o2-tpc-reco-workflow " --disable-ctp-lumi-request --input-type digitizer --output-type \"tracks,disable-writer\" --disable-mc --pipeline tpc-zsEncoder:20,tpc-tracker:8 ${GPU_CONFIG} ${REMAP}" "${RECO_CONFIG}"
+add_W o2-tpc-scaler-workflow "--disable-IDC-scalers --disable-ctp-lumi-request"
+add_W o2-tpc-reco-workflow "--input-type digitizer --output-type \"tracks,disable-writer\" --disable-mc --pipeline tpc-zsEncoder:20,tpc-tracker:8 ${GPU_CONFIG} ${REMAP}" "${RECO_CONFIG}"
 add_W o2-tpc-laser-track-filter "" "" 0
 add_W o2-tpc-calib-laser-tracks "--use-filtered-tracks ${EXTRA_CONFIG_TRACKS} --min-tfs=${min_tracks}"
 add_W o2-tpc-calib-pad-raw " ${EXTRA_CONFIG}" "${CALIB_CONFIG}"

DATA/production/common/readAO2Ds.C

@@ -35,6 +35,17 @@ int readAO2Ds(const char* filename = "AO2D.root")
         std::cout << onameKeyInDir.Data() << std::endl;
       }
       TTree* t = (TTree*)d->Get(onameKeyInDir.Data());
+      // Check that every branch has the same number of entries as the tree itself.
+      // A mismatch indicates a corrupted or truncated write.
+      long long treeEntries = t->GetEntries();
+      for (auto* b : *t->GetListOfBranches()) {
+        auto* br = (TBranch*)b;
+        if (br->GetEntries() != treeEntries) {
+          printf("MISMATCH %s/%s branch %s: tree=%lld branch=%lld\n",
+                 onameKeyInFile.Data(), onameKeyInDir.Data(), br->GetName(), treeEntries, br->GetEntries());
+          retCode |= 4;
+        }
+      }
       if (onameKeyInDir.BeginsWith("O2track") && !onameKeyInDir.Contains("O2tracked") && !onameKeyInDir.Contains("O2trackqa")) {
         vectNEntriesPerTree.push_back({onameKeyInDir.Data(), t->GetEntries()});
       }

DATA/production/production.desc

@@ -1,3 +1,3 @@
 synchronous-workflow: "O2PDPSuite" reco,128,126,"GPU_NUM_MEM_REG_CALLBACKS=5 SYNCMODE=1 NUMAGPUIDS=1 NUMAID=0 production/dpl-workflow.sh" reco,128,126,"GPU_NUM_MEM_REG_CALLBACKS=5 SYNCMODE=1 NUMAGPUIDS=1 NUMAID=1 production/dpl-workflow.sh"
 synchronous-workflow-1numa: "O2PDPSuite" reco,128,126,"SYNCMODE=1 production/dpl-workflow.sh"
-synchronous-workflow-calib: "O2PDPSuite" reco,128,126,"GPU_NUM_MEM_REG_CALLBACKS=5 SYNCMODE=1 NUMAGPUIDS=1 NUMAID=0 WORKFLOW_PARAMETERS+=,CALIB_PROXIES production/dpl-workflow.sh" reco,128,126,"GPU_NUM_MEM_REG_CALLBACKS=5 SYNCMODE=1 NUMAGPUIDS=1 NUMAID=1 WORKFLOW_PARAMETERS+=,CALIB_PROXIES production/dpl-workflow.sh" calib,32,"AGGREGATOR_TASKS=BARREL_TF SYNCMODE=1 SHMSIZE=68719476736 WORKFLOW_PARAMETERS+=,CALIB_PROXIES production/aggregator-workflow.sh" calib,32,"AGGREGATOR_TASKS=BARREL_SPORADIC SYNCMODE=1 SHMSIZE=68719476736 WORKFLOW_PARAMETERS+=,CALIB_PROXIES production/aggregator-workflow.sh" calib,16,"AGGREGATOR_TASKS=CALO_TF SYNCMODE=1 SHMSIZE=34359738368 WORKFLOW_PARAMETERS+=,CALIB_PROXIES production/aggregator-workflow.sh" calib,16,"AGGREGATOR_TASKS=CALO_SPORADIC SYNCMODE=1 SHMSIZE=34359738368 WORKFLOW_PARAMETERS+=,CALIB_PROXIES production/aggregator-workflow.sh" calib,16,"AGGREGATOR_TASKS=MUON_TF SYNCMODE=1 SHMSIZE=34359738368 WORKFLOW_PARAMETERS+=,CALIB_PROXIES production/aggregator-workflow.sh" calib,16,"AGGREGATOR_TASKS=MUON_SPORADIC SYNCMODE=1 SHMSIZE=34359738368 WORKFLOW_PARAMETERS+=,CALIB_PROXIES production/aggregator-workflow.sh" calib,16,"AGGREGATOR_TASKS=FORWARD_TF SYNCMODE=1 SHMSIZE=34359738368 WORKFLOW_PARAMETERS+=,CALIB_PROXIES production/aggregator-workflow.sh" calib,16,"AGGREGATOR_TASKS=FORWARD_SPORADIC SYNCMODE=1 SHMSIZE=34359738368 WORKFLOW_PARAMETERS+=,CALIB_PROXIES production/aggregator-workflow.sh" calib,128,"AGGREGATOR_TASKS=TPC_IDCBOTH_SAC SHMSIZE=137438953472 WORKFLOW_PARAMETERS+=,CALIB_PROXIES production/aggregator-workflow.sh"
+synchronous-workflow-calib: "O2PDPSuite" reco,128,126,"GPU_NUM_MEM_REG_CALLBACKS=5 SYNCMODE=1 NUMAGPUIDS=1 NUMAID=0 WORKFLOW_PARAMETERS+=,CALIB_PROXIES production/dpl-workflow.sh" reco,128,126,"GPU_NUM_MEM_REG_CALLBACKS=5 SYNCMODE=1 NUMAGPUIDS=1 NUMAID=1 WORKFLOW_PARAMETERS+=,CALIB_PROXIES production/dpl-workflow.sh" calib,32,"AGGREGATOR_TASKS=BARREL_TF SYNCMODE=1 SHMSIZE=68719476736 WORKFLOW_PARAMETERS+=,CALIB_PROXIES production/aggregator-workflow.sh" calib,32,"AGGREGATOR_TASKS=BARREL_SPORADIC SYNCMODE=1 SHMSIZE=68719476736 WORKFLOW_PARAMETERS+=,CALIB_PROXIES production/aggregator-workflow.sh" calib,16,"AGGREGATOR_TASKS=CALO_TF SYNCMODE=1 SHMSIZE=34359738368 WORKFLOW_PARAMETERS+=,CALIB_PROXIES production/aggregator-workflow.sh" calib,16,"AGGREGATOR_TASKS=CALO_SPORADIC SYNCMODE=1 SHMSIZE=34359738368 WORKFLOW_PARAMETERS+=,CALIB_PROXIES production/aggregator-workflow.sh" calib,16,"AGGREGATOR_TASKS=MUON_TF SYNCMODE=1 SHMSIZE=34359738368 WORKFLOW_PARAMETERS+=,CALIB_PROXIES production/aggregator-workflow.sh" calib,16,"AGGREGATOR_TASKS=MUON_SPORADIC SYNCMODE=1 SHMSIZE=34359738368 WORKFLOW_PARAMETERS+=,CALIB_PROXIES production/aggregator-workflow.sh" calib,16,"AGGREGATOR_TASKS=FORWARD_TF SYNCMODE=1 SHMSIZE=34359738368 WORKFLOW_PARAMETERS+=,CALIB_PROXIES production/aggregator-workflow.sh" calib,16,"AGGREGATOR_TASKS=FORWARD_SPORADIC SYNCMODE=1 SHMSIZE=34359738368 WORKFLOW_PARAMETERS+=,CALIB_PROXIES production/aggregator-workflow.sh" calib,128,"AGGREGATOR_TASKS=TPC_IDCBOTH_SAC SHMSIZE=137438953472 WORKFLOW_PARAMETERS+=,CALIB_PROXIES production/aggregator-workflow.sh" calib,128,"AGGREGATOR_TASKS=TPC_CMV SHMSIZE=137438953472 WORKFLOW_PARAMETERS+=,CALIB_PROXIES production/aggregator-workflow.sh"

DATA/production/workflow-multiplicities.sh

@@ -301,8 +301,8 @@ if [[ "$HIGH_RATE_PP" == "1" ]]; then
   : ${CUT_RANDOM_FRACTION_ITS:=0.97}
 elif [[ $BEAMTYPE == "PbPb" ]]; then
   : ${CUT_RANDOM_FRACTION_ITS:=-1}
-  : ${CUT_MULT_MIN_ITS:=100}
-  : ${CUT_MULT_MAX_ITS:=200}
+  : ${CUT_MULT_MIN_ITS:=0}
+  : ${CUT_MULT_MAX_ITS:=400}
   : ${CUT_MULT_VTX_ITS:=20}
 else
   : ${CUT_RANDOM_FRACTION_ITS:=0.95}

MC/bin/o2dpg_sim_workflow.py

@@ -126,6 +126,8 @@
 #  help='Treat smaller sensors in a single digitization')
 parser.add_argument('--pregenCollContext', action='store_true', help=argparse.SUPPRESS) # Now the default, giving this option or not makes not difference. We keep it for backward compatibility
 parser.add_argument('--data-anchoring', type=str, default='', help="Take collision contexts (from data) stored in this path")
+parser.add_argument('--aod-output-folder', type=str, default='', help="Force this AOD folder in the AOD producer")
+parser.add_argument('--aod-parent-file', type=str, default='', help="Link this as parent file in the AOD")
 parser.add_argument('--no-combine-smaller-digi', action='store_true', help=argparse.SUPPRESS)
 parser.add_argument('--no-combine-dpl-devices', action='store_true', help=argparse.SUPPRESS)
 parser.add_argument('--no-mc-labels', action='store_true', default=False, help=argparse.SUPPRESS)
@@ -672,7 +674,7 @@ def getDPL_global_options(bigshm=False, ccdbbackend=True, runcommand=True):
       f'--timestamp {args.timestamp}',
       f'--import-external {args.data_anchoring}' if len(args.data_anchoring) > 0 else None,
       '--bcPatternFile ccdb',
-      ' --nontrivial-mu-distribution ccdb://https://alice-ccdb.cern.ch/FTO/Calib/EventsPerBc',
+      ' --nontrivial-mu-distribution ccdb://https://alice-ccdb.cern.ch/FT0/Calib/EventsPerBc',
       f'--QEDinteraction {qedspec}' if includeQED else None
       ], configname = 'precollcontext')
 workflow['stages'].append(PreCollContextTask)
@@ -1796,6 +1798,16 @@ def getDigiTaskName(det):
    if created_by_option != '':
       created_by_option += ' ' + aod_creator
 
+   aod_parent_option = option_if_available('o2-aod-producer-workflow', '--aod-parent', envfile=async_envfile)
+   if aod_parent_option != '' and len(args.aod_parent_file) > 0:
+      aod_parent_option += ' ' + args.aod_parent_file
+   else:
+      aod_parent_option = ''
+
+   aod_timeframe_id = f"${{ALIEN_PROC_ID}}{aod_df_id}" if not args.run_anchored else ""
+   if len(args.aod_output_folder) > 0:
+       aod_timeframe_id = args.aod_output_folder
+
    AODtask = createTask(name='aod_'+str(tf), needs=aodneeds, tf=tf, cwd=timeframeworkdir, lab=["AOD"], mem='4000', cpu='1')
    AODtask['cmd'] = ('','ln -nfs ../bkg_Kine.root . ;')[doembedding]
    AODtask['cmd'] += '[ -f AO2D.root ] && rm AO2D.root; '
@@ -1812,12 +1824,13 @@ def getDigiTaskName(det):
       "--anchor-pass ${ALIEN_JDL_LPMANCHORPASSNAME:-unknown}",
       "--anchor-prod ${ALIEN_JDL_LPMANCHORPRODUCTION:-unknown}",
       "--reco-pass ${ALIEN_JDL_LPMPASSNAME:-unknown}",
+      aod_parent_option,
       created_by_option,
       "--combine-source-devices" if not args.no_combine_dpl_devices else "",
       "--disable-mc" if args.no_mc_labels else "",
       "--enable-truncation 0" if environ.get("O2DPG_AOD_NOTRUNCATE") or environ.get("ALIEN_JDL_O2DPG_AOD_NOTRUNCATE") else "",
       "--disable-strangeness-tracker" if args.no_strangeness_tracking else "",
-      f"--aod-timeframe-id ${{ALIEN_PROC_ID}}{aod_df_id}" if not args.run_anchored else "",
+      f"--aod-timeframe-id {aod_timeframe_id}" if len(aod_timeframe_id) > 0 else ""
    ])
    # Consider in future: AODtask['disable_alternative_reco_software'] = True # do not apply reco software here (we prefer latest aod converter)
    workflow['stages'].append(AODtask)