mcfix

CODEOWNERS

@@ -17,9 +17,10 @@
 /DATA/testing/detectors/TRD
 /DATA/testing/detectors/ZDC
 
-/MC @sawenzel @chiarazampolli @jackal1-66
+/MC @sawenzel @jackal1-66
 
 # PWG experts
+/MC/*/ALICE3 @njacazio
 /MC/*/PWGHF @AliceO2Group/reviewers-pwg-hf
 /MC/*/PWGLF @AliceO2Group/reviewers-pwg-lf
 /MC/*/PWGEM @AliceO2Group/reviewers-pwg-em

DATA/common/README.md

@@ -1,51 +1 @@
-The `setenv-sh` script sets the following environment options
-* `NTIMEFRAMES`: Number of time frames to process.
-* `TFDELAY`: Delay in seconds between publishing time frames (1 / rate).
-* `NGPUS`: Number of GPUs to use, data distributed round-robin.
-* `GPUTYPE`: GPU Tracking backend to use, can be CPU / CUDA / HIP / OCL / OCL2.
-* `SHMSIZE`: Size of the global shared memory segment.
-* `DDSHMSIZE`: Size of shared memory unmanaged region for DataDistribution Input.
-* `GPUMEMSIZE`: Size of allocated GPU memory (if GPUTYPE != CPU)
-* `HOSTMEMSIZE`: Size of allocated host memory for GPU reconstruction (0 = default).
-  * For `GPUTYPE = CPU`: TPC Tracking scratch memory size. (Default 0 -> dynamic allocation.)
-  * Otherwise : Size of page-locked host memory for GPU processing. (Defauls 0 -> 1 GB.)
-* `CREATECTFDICT`: Create CTF dictionary.
-* `SAVECTF`: Save the CTF to a root file.
-  * 0: Read `ctf_dictionary.root` as input.
-  * 1: Create `ctf_dictionary.root`. Note that this was already done automatically if the raw data was simulated with `full_system_test.sh`.
-* `SYNCMODE`: Run only reconstruction steps of the synchronous reconstruction.
-  * Note that there is no `ASYNCMODE` but instead the `CTFINPUT` option already enforces asynchronous processing.
-* `NUMAGPUIDS`: NUMAID-aware GPU id selection. Needed for the full EPN configuration with 8 GPUs, 2 NUMA domains, 4 GPUs per domain.
-  In this configuration, 2 instances of `dpl-workflow.sh` must run in parallel.
-  To be used in combination with `NUMAID` to select the id per workflow.
-  `start_tmux.sh` will set up these variables automatically.
-* `NUMAID`: SHM segment id to use for shipping data as well as set of GPUs to use (use `0` / `1` for 2 NUMA domains, 0 = GPUS `0` to `NGPUS - 1`, 1 = GPUS `NGPUS` to `2 * NGPUS - 1`)
-* 0: Runs all reconstruction steps, of sync and of async reconstruction, using raw data input.
-* 1: Runs only the steps of synchronous reconstruction, using raw data input.
-* `EXTINPUT`: Receive input from raw FMQ channel instead of running o2-raw-file-reader.
-  * 0: `dpl-workflow.sh` can run as standalone benchmark, and will read the input itself.
-  * 1: To be used in combination with either `datadistribution.sh` or `raw-reader.sh` or with another DataDistribution instance.
-* `CTFINPUT`: Read input from CTF ROOT file. This option is incompatible to EXTINPUT=1. The CTF ROOT file can be stored via SAVECTF=1.
-* `NHBPERTF`: Time frame length (in HBF)
-* `GLOBALDPLOPT`: Global DPL workflow options appended to o2-dpl-run.
-* `EPNPIPELINES`: Set default EPN pipeline multiplicities.
-  Normally the workflow will start 1 dpl device per processor.
-  For some of the CPU parts, this is insufficient to keep step with the GPU processing rate, e.g. one ITS-TPC matcher on the CPU is slower than the TPC tracking on multiple GPUs.
-  This option adds some multiplicies for CPU processes using DPL's pipeline feature.
-  The settings were tuned for EPN processing with 4 GPUs (i.e. the default multiplicities are per NUMA domain).
-  The multiplicities are scaled with the `NGPUS` setting, i.e. with 1 GPU only 1/4th are applied.
-  You can pass an option different to 1, and than it will be applied as factor on top of the multiplicities.
-  It is auto-selected by `start-tmux.sh`.
-* `SEVERITY`: Log verbosity (e.g. info or error, default: info)
-* `INFOLOGGER_SEVERITY`: Min severity for messages sent to Infologger. (default: `$SEVERITY`)
-* `SHMTHROW`: Throw exception when running out of SHM memory.
-  It is suggested to leave this enabled (default) on tests on the laptop to get an actual error when it runs out of memory.
-  This is disabled in `start_tmux.sh`, to avoid breaking the processing while there is a chance that another process might free memory and we can continue.
-* `NORATELOG`: Disable FairMQ Rate Logging.
-* `INRAWCHANNAME`: FairMQ channel name used by the raw proxy, must match the name used by DataDistribution.
-* `WORKFLOWMODE`: run (run the workflow (default)), print (print the command to stdout), dds (create partial DDS topology)
-* `FILEWORKDIR`: directory for all input / output files. E.g. grp / geometry / dictionaries etc. are read from here, and dictionaries / ctf / etc. are written to there.
-  Some files have more fine grained control via other environment variables (e.g. to store the CTF to somewhere else). Such variables are initialized to `$FILEWORKDIR` by default but can be overridden.
-* `EPNSYNCMODE`: Specify that this is a workflow running on the EPN for synchronous processing, e.g. logging goes to InfoLogger, DPL metrics to to the AliECS monitoring, etc.
-* `BEAMTYPE`: Beam type, must be PbPb, pp, pPb, cosmic, technical.
-* `IS_SIMULATED_DATA` : 1 for MC data, 0 for RAW data.
+For a reference to available env-variables, please check https://github.com/AliceO2Group/AliceO2/blob/dev/prodtests/full-system-test/documentation/env-variables.md

DATA/common/gen_topo_helper_functions.sh

@@ -59,6 +59,11 @@ has_secvtx_source()
   [[ $SVERTEXING_SOURCES =~ (^|,)"ALL"(,|$) ]] || [[ $SVERTEXING_SOURCES =~ (^|,)"$1"(,|$) ]]
 }
 
+has_detector_in_secvtx_sources()
+{
+  [[ $SVERTEXING_SOURCES =~ (^|,)"ALL"(,|$) ]] || [[ $SVERTEXING_SOURCES =~ (^|,|-)"$1"(-|,|$) ]]
+}
+
 has_detector_qc()
 {
   has_detector $1 && [[ $WORKFLOW_DETECTORS_QC =~ (^|,)"$1"(,|$) ]]

DATA/common/setenv.sh

@@ -91,7 +91,7 @@ if [[ -z "${FILEWORKDIR:-}" ]];    then export FILEWORKDIR=`pwd`; fi           #
 if [[ -z "${FILEWORKDIRRUN:-}" ]]; then export FILEWORKDIRRUN=$FILEWORKDIR; fi # directory where to find the run-related files (grp, collision context)
 if [[ -z "${RAWINPUTDIR:-}" ]];    then export RAWINPUTDIR=$FILEWORKDIR; fi    # Directory where to find input files (raw files / raw tf files / ctf files)
 if [[ -z "${EPNSYNCMODE:-}" ]];    then export EPNSYNCMODE=0; fi               # Is this workflow supposed to run on EPN for sync processing? Will enable InfoLogger / metrics / fetching QC JSONs from consul...
-if [[ -z "${BEAMTYPE:-}" ]];       then export BEAMTYPE=PbPb; fi               # Beam type, must be PbPb, pp, pPb, cosmic, technical
+if [[ -z "${BEAMTYPE:-}" ]];       then export BEAMTYPE=PbPb; fi               # Beam type, must be PbPb, pp, pPb, pO, Op, OO, NeNe cosmic, technical
 if [[ -z "${RUNTYPE:-}" ]];        then export RUNTYPE=Standalone; fi          # Run Type, standalone for local tests, otherwise PHYSICS, COSMICS, TECHNICAL, SYNTHETIC
 if [[ -z "${IS_SIMULATED_DATA:-}" && $RUNTYPE == "SYNTHETIC" ]]; then export IS_SIMULATED_DATA=1; fi # For SYNTHETIC runs we always process simulated data
 if [[ -z "${IS_SIMULATED_DATA:-}" && ( $RUNTYPE == "PHYSICS" || $RUNTYPE == "COSMICS" ) ]]; then export IS_SIMULATED_DATA=0; fi # For PHYSICS runs we always process simulated data
@@ -103,6 +103,12 @@ if [[ -z "${EPN2EOS_METAFILES_DIR:-}" ]]; then EPN2EOS_METAFILES_DIR="/dev/null"
 if [[ -z "${DCSCCDBSERVER:-}" ]];  then export DCSCCDBSERVER="http://alio2-cr1-flp199-ib:8083"; fi # server for transvering calibration data to DCS
 if [[ -z "${DCSCCDBSERVER_PERS:-}" ]]; then export DCSCCDBSERVER_PERS="http://alio2-cr1-flp199-ib:8084"; fi # persistent server for transvering calibration data to DCS
 
+if [[ $BEAMTYPE == "pO" ]] || [[ $BEAMTYPE == "Op" ]] || [[ $BEAMTYPE == "Op" ]] || [[ $BEAMTYPE == "OO" ]] || [[ $BEAMTYPE == "NeNe" ]] ; then
+  export LIGHTNUCLEI=1
+else
+  export LIGHTNUCLEI=0
+fi
+
 if [[ $EPNSYNCMODE == 0 ]]; then
   if [[ -z "${SHMSIZE:-}" ]];       then export SHMSIZE=$(( 8 << 30 )); fi    # Size of shared memory for messages
   if [[ -z "${NGPUS:-}" ]];         then export NGPUS=1; fi                   # Number of GPUs to use, data distributed round-robin
@@ -144,7 +150,7 @@ if [[ `uname` == Darwin ]]; then export UDS_PREFIX=; else export UDS_PREFIX="@";
 
 # Env variables required for workflow setup
 if [[ $SYNCMODE == 1 ]]; then
-  if [[ -z "${WORKFLOW_DETECTORS_MATCHING+x}" ]]; then export WORKFLOW_DETECTORS_MATCHING="ITSTPC,ITSTPCTRD,ITSTPCTOF,ITSTPCTRDTOF,PRIMVTX"; fi # Select matchings that are enabled in sync mode
+  if [[ -z "${WORKFLOW_DETECTORS_MATCHING+x}" ]]; then export WORKFLOW_DETECTORS_MATCHING="ITSTPC,ITSTPCTRD,ITSTPCTOF,ITSTPCTRDTOF,PRIMVTX,SECVTX"; fi # Select matchings that are enabled in sync mode
 else
   if [[ -z "${WORKFLOW_DETECTORS_MATCHING+x}" ]]; then export WORKFLOW_DETECTORS_MATCHING="ALL"; fi # All matching / vertexing enabled in async mode
 fi
@@ -158,7 +164,7 @@ DISABLE_ROOT_INPUT="--disable-root-input"
 # Special detector related settings
 if [[ -z "${TPC_CORR_SCALING:-}" ]]; then # TPC corr.map lumi scaling options, any combination of --lumi-type <0,1,2> --corrmap-lumi-mode <0,1>  and TPCCorrMap... configurable param
  TPC_CORR_SCALING=
- if [[ $BEAMTYPE == "pp" ]] && has_detector CTP; then TPC_CORR_SCALING+="--lumi-type 1"; fi
+ if [[ $BEAMTYPE == "pp" || $LIGHTNUCLEI == "1" ]] && has_detector CTP; then TPC_CORR_SCALING+="--lumi-type 1"; fi
  if [[ $BEAMTYPE == "PbPb" ]] && has_detector CTP; then TPC_CORR_SCALING+="--lumi-type 1 TPCCorrMap.lumiInstFactor=2.414"; fi
  if [[ $BEAMTYPE == "cosmic" ]]; then TPC_CORR_SCALING=" TPCCorrMap.lumiMean=-1;"; fi # for COSMICS we disable all corrections
  export TPC_CORR_SCALING=$TPC_CORR_SCALING
@@ -189,6 +195,7 @@ TRD_SOURCES=
 TOF_SOURCES=
 HMP_SOURCES=
 TRACK_SOURCES=
+: ${TRACK_SOURCES_GLO:=}
 has_detectors_reco ITS TPC && has_detector_matching ITSTPC && add_comma_separated TRACK_SOURCES "ITS-TPC"
 has_detectors_reco TPC TRD && has_detector_matching TPCTRD && { add_comma_separated TRD_SOURCES TPC; add_comma_separated TRACK_SOURCES "TPC-TRD"; }
 has_detectors_reco ITS TPC TRD && has_detector_matching ITSTPC && has_detector_matching ITSTPCTRD && { add_comma_separated TRD_SOURCES ITS-TPC; add_comma_separated TRACK_SOURCES "ITS-TPC-TRD"; }
@@ -205,6 +212,8 @@ has_detectors_reco HMP TPC TOF && has_detector_matching TPCTOF && add_comma_sepa
 has_detectors_reco HMP TPC TRD TOF && has_detector_matching TPCTRD && has_detector_matching TPCTRDTOF && add_comma_separated HMP_SOURCES "TPC-TRD-TOF"
 has_detectors_reco MFT MCH && has_detector_matching MFTMCH && add_comma_separated TRACK_SOURCES "MFT-MCH"
 has_detectors_reco MCH MID && has_detector_matching MCHMID && add_comma_separated TRACK_SOURCES "MCH-MID"
+[[ "0$TRACK_SOURCES_GLO" == "0" ]] && TRACK_SOURCES_GLO=$TRACK_SOURCES
+
 for det in `echo $LIST_OF_DETECTORS | sed "s/,/ /g"`; do
   if [[ $LIST_OF_ASYNC_RECO_STEPS =~ (^| )${det}( |$) ]]; then
     has_detector ${det} && has_processing_step ${det}_RECO && add_comma_separated TRACK_SOURCES "$det"
@@ -235,12 +244,6 @@ fi
 [[ ! -z $VERTEXING_SOURCES ]] && PVERTEX_CONFIG+=" --vertexing-sources $VERTEXING_SOURCES"
 [[ ! -z $VERTEX_TRACK_MATCHING_SOURCES ]] && PVERTEX_CONFIG+=" --vertex-track-matching-sources $VERTEX_TRACK_MATCHING_SOURCES"
 
-if [[ -z ${SVERTEXING_SOURCES:-} ]]; then
-  SVERTEXING_SOURCES="$VERTEXING_SOURCES"
-elif [[ "${SVERTEXING_SOURCES^^}" == "NONE" ]]; then
-  SVERTEXING_SOURCES=
-fi
-
 # this option requires well calibrated timing beween different detectors, at the moment suppress it
 #has_detector_reco FT0 && PVERTEX_CONFIG+=" --validate-with-ft0"
 

DATA/common/setenv_calib.sh

@@ -17,7 +17,7 @@ if has_detector_calib ITS ; then CAN_DO_CALIB_ITS_DEADMAP_TIME=1; else CAN_DO_CA
 if has_detector_calib MFT ; then CAN_DO_CALIB_MFT_DEADMAP_TIME=1; else CAN_DO_CALIB_MFT_DEADMAP_TIME=0; fi
 if has_detector_calib TOF && has_detector_reco TOF; then CAN_DO_CALIB_TOF_DIAGNOSTICS=1; CAN_DO_CALIB_TOF_INTEGRATEDCURR=1; else CAN_DO_CALIB_TOF_DIAGNOSTICS=0; CAN_DO_CALIB_TOF_INTEGRATEDCURR=0; fi
 if has_detector_calib TOF && has_detector_reco TOF && ( ( has_detectors_reco ITS TPC && has_detector_matching ITSTPCTOF ) || ( has_detectors_reco ITS TPC TRD && has_detector_matching ITSTPCTRDTOF ) ); then CAN_DO_CALIB_TOF_LHCPHASE=1; CAN_DO_CALIB_TOF_CHANNELOFFSETS=1; else CAN_DO_CALIB_TOF_LHCPHASE=0; CAN_DO_CALIB_TOF_CHANNELOFFSETS=0; fi
-if has_detector_calib TPC && has_detectors ITS TPC TOF TRD && has_detector_matching ITSTPCTRDTOF; then CAN_DO_CALIB_TPC_SCDCALIB=1; else CAN_DO_CALIB_TPC_SCDCALIB=0; fi
+if has_detector_calib TPC && has_detectors ITS TPC && has_detector_matching ITSTPC; then CAN_DO_CALIB_TPC_SCDCALIB=1; else CAN_DO_CALIB_TPC_SCDCALIB=0; fi
 if has_detector_calib TPC && has_processing_step TPC_DEDX; then CAN_DO_CALIB_TPC_TIMEGAIN=1; CAN_DO_CALIB_TPC_RESPADGAIN=1; else CAN_DO_CALIB_TPC_TIMEGAIN=0; CAN_DO_CALIB_TPC_RESPADGAIN=0; fi
 if has_detector_calib TPC && has_detectors ITS TPC && has_detector_matching ITSTPC; then CAN_DO_CALIB_TPC_VDRIFTTGL=1; else CAN_DO_CALIB_TPC_VDRIFTTGL=0; fi
 if has_detector_calib TPC; then CAN_DO_CALIB_TPC_IDC=1; CAN_DO_CALIB_TPC_SAC=1; else CAN_DO_CALIB_TPC_IDC=0; CAN_DO_CALIB_TPC_SAC=0; fi
@@ -51,17 +51,17 @@ if [[ $BEAMTYPE != "cosmic" ]] || [[ ${FORCECALIBRATIONS:-} == 1 ]] ; then # Cal
   if [[ $CAN_DO_CALIB_PRIMVTX_MEANVTX == 1 ]]; then
     if [[ -z ${CALIB_PRIMVTX_MEANVTX+x} ]]; then CALIB_PRIMVTX_MEANVTX=1; fi
   fi
-  
+
   # calibrations for ITS
   if [[ $CAN_DO_CALIB_ITS_DEADMAP_TIME == 1 ]]; then
     if [[ -z ${CALIB_ITS_DEADMAP_TIME+x} ]]; then CALIB_ITS_DEADMAP_TIME=1; fi
   fi
-  
+
   # calibrations for MFT
   if [[ $CAN_DO_CALIB_MFT_DEADMAP_TIME == 1 ]]; then
     if [[ -z ${CALIB_MFT_DEADMAP_TIME+x} ]]; then CALIB_MFT_DEADMAP_TIME=1; fi
   fi
-  
+
   # calibrations for TOF
   if [[ $CAN_DO_CALIB_TOF_DIAGNOSTICS == 1 ]]; then
     if [[ -z ${CALIB_TOF_DIAGNOSTICS+x} ]]; then CALIB_TOF_DIAGNOSTICS=1; fi
@@ -260,14 +260,15 @@ if [[ -z ${CALIBDATASPEC_BARREL_TF:-} ]]; then
 
   # MFT
   if [[ $CALIB_MFT_DEADMAP_TIME == 1 ]]; then add_semicolon_separated CALIBDATASPEC_BARREL_TF "mftChipStatus:MFT/CHIPSSTATUS/0"; fi
-  
+
   # TOF
   if [[ $CALIB_TOF_LHCPHASE == 1 ]] || [[ $CALIB_TOF_CHANNELOFFSETS == 1 ]]; then add_semicolon_separated CALIBDATASPEC_BARREL_TF "calibTOF:TOF/CALIBDATA/0"; fi
   if [[ $CALIB_TOF_DIAGNOSTICS == 1 ]]; then add_semicolon_separated CALIBDATASPEC_BARREL_TF "diagWords:TOF/DIAFREQ/0"; fi
 
   # TPC
   if [[ $CALIB_TPC_SCDCALIB == 1 ]]; then
     add_semicolon_separated CALIBDATASPEC_BARREL_TF "unbinnedTPCResiduals:GLO/UNBINNEDRES/0"
+    add_semicolon_separated CALIBDATASPEC_BARREL_TF "detectorInfoResiduals:GLO/DETINFORES/0"
     add_semicolon_separated CALIBDATASPEC_BARREL_TF "trackReferences:GLO/TRKREFS/0"
   fi
   if [[ $CALIB_TPC_SCDCALIB == 1 ]] && [[ ${CALIB_TPC_SCDCALIB_SENDTRKDATA:-} == "1" ]]; then add_semicolon_separated CALIBDATASPEC_BARREL_TF "tpcInterpTrkData:GLO/TRKDATA/0"; fi