Merge branch 'dev' into stable-sync

Author: ehellbar

CCDB/include/CCDB/BasicCCDBManager.h

@@ -115,6 +115,10 @@ class CCDBManagerInstance
     return getForTimeStamp<T>(path, timestamp);
   }
 
+  /// retrieve an object of type T from CCDB as stored under path and using the timestamp in the middle of the run + metadata. The run number is provided separately to conform to typical analysis use (in which case metadata does not include runNumber)
+  template <typename T>
+  T* getSpecificForRun(std::string const& path, int runNumber, MD metaData = MD());
+
   /// detect online processing modes (i.e. CCDB objects may be updated in the lifetime of the manager)
   bool isOnline() const { return mDeplMode == o2::framework::DeploymentMode::OnlineAUX || mDeplMode == o2::framework::DeploymentMode::OnlineDDS || mDeplMode == o2::framework::DeploymentMode::OnlineECS; }
 
@@ -317,6 +321,14 @@ T* CCDBManagerInstance::getForTimeStamp(std::string const& path, long timestamp)
 
 template <typename T>
 T* CCDBManagerInstance::getForRun(std::string const& path, int runNumber, bool setRunMetadata)
+{
+  auto metaData = setRunMetadata ? MD{{"runNumber", std::to_string(runNumber)}} : MD{};
+  mMetaData = metaData;
+  return getSpecificForRun<T>(path, runNumber, metaData);
+}
+
+template <typename T>
+T* CCDBManagerInstance::getSpecificForRun(std::string const& path, int runNumber, MD metaData)
 {
   auto [start, stop] = getRunDuration(runNumber);
   if (start < 0 || stop < 0) {
@@ -325,8 +337,7 @@ T* CCDBManagerInstance::getForRun(std::string const& path, int runNumber, bool s
     }
     return nullptr;
   }
-  mMetaData = setRunMetadata ? MD{{"runNumber", std::to_string(runNumber)}} : MD{};
-  return getForTimeStamp<T>(path, start / 2 + stop / 2);
+  return getSpecific<T>(path, start / 2 + stop / 2, metaData);
 }
 
 class BasicCCDBManager : public CCDBManagerInstance

CCDB/include/CCDB/CcdbApi.h

@@ -392,6 +392,9 @@ class CcdbApi //: public DatabaseInterface
   // Loads files from alien and cvmfs into given destination.
   bool loadLocalContentToMemory(o2::pmr::vector<char>& dest, std::string& url) const;
 
+  // add annotated flattened headers in the end of the blob
+  static void appendFlatHeader(o2::pmr::vector<char>& dest, const std::map<std::string, std::string>& headers);
+
   // the failure to load the file to memory is signaled by 0 size and non-0 capacity
   static bool isMemoryFileInvalid(const o2::pmr::vector<char>& v) { return v.size() == 0 && v.capacity() > 0; }
   template <typename T>
@@ -610,6 +613,16 @@ class CcdbApi //: public DatabaseInterface
     return getSnapshotDir(topdir, path) + '/' + sfile;
   }
 
+  template <typename MAP> // can be either std::map or std::multimap
+  static size_t getFlatHeaderSize(const MAP& Headers)
+  {
+    size_t hsize = sizeof(int) + sizeof(FlatHeaderAnnot); // annotation size
+    for (auto& h : Headers) {
+      hsize += h.first.length() + h.second.length() + 2; // 2*(string_buffer + terminating null character)
+    }
+    return hsize;
+  }
+
   // tmp helper and single point of entry for a CURL perform call
   // helps to switch between easy handle perform and multi handles in a single place
   CURLcode CURL_perform(CURL* handle) const;
@@ -632,6 +645,8 @@ class CcdbApi //: public DatabaseInterface
   size_t mCurlTimeoutDownload = 15; // download timeout in seconds, can be configured via ALICEO2_CCDB_CURL_TIMEOUT_DOWNLOAD, updated according to the deployment mode
   size_t mCurlTimeoutUpload = 15;   // upload timeout in seconds, can be configured via ALICEO2_CCDB_CURL_TIMEOUT_UPLOAD, updated according to the deployment mode
 
+  static constexpr char FlatHeaderAnnot[] = "$HEADER$"; // annotation for flat header
+
   ClassDefNV(CcdbApi, 1);
 };
 

CCDB/src/CcdbApi.cxx

@@ -1687,12 +1687,15 @@ void CcdbApi::scheduleDownload(RequestContext& requestContext, size_t* requestCo
     ho.counter++;
     try {
       if (chunk.capacity() < chunk.size() + realsize) {
+        // estimate headers size when converted to annotated text string
+        const char hannot[] = "header";
+        size_t hsize = getFlatHeaderSize(ho.header);
         auto cl = ho.header.find("Content-Length");
         if (cl != ho.header.end()) {
           size_t sizeFromHeader = std::stol(cl->second);
-          sz = std::max(chunk.size() * (sizeFromHeader ? 1 : 2) + realsize, sizeFromHeader);
+          sz = hsize + std::max(chunk.size() * (sizeFromHeader ? 1 : 2) + realsize, sizeFromHeader);
         } else {
-          sz = std::max(chunk.size() * 2, chunk.size() + realsize);
+          sz = hsize + std::max(chunk.size() * 2, chunk.size() + realsize);
           // LOGP(debug, "SIZE IS NOT IN HEADER, allocate {}", sz);
         }
         chunk.reserve(sz);
@@ -1885,6 +1888,25 @@ void CcdbApi::loadFileToMemory(o2::pmr::vector<char>& dest, std::string const& p
   vectoredLoadFileToMemory(contexts);
 }
 
+void CcdbApi::appendFlatHeader(o2::pmr::vector<char>& dest, const std::map<std::string, std::string>& headers)
+{
+  size_t hsize = getFlatHeaderSize(headers), cnt = dest.size();
+  dest.resize(cnt + hsize);
+  auto addString = [&dest, &cnt](const std::string& s) {
+    for (char c : s) {
+      dest[cnt++] = c;
+    }
+    dest[cnt++] = 0;
+  };
+
+  for (auto& h : headers) {
+    addString(h.first);
+    addString(h.second);
+  }
+  *reinterpret_cast<int*>(&dest[cnt]) = hsize;                                     // store size
+  std::memcpy(&dest[cnt + sizeof(int)], FlatHeaderAnnot, sizeof(FlatHeaderAnnot)); // annotate the flattened headers map
+}
+
 void CcdbApi::navigateSourcesAndLoadFile(RequestContext& requestContext, int& fromSnapshot, size_t* requestCounter) const
 {
   LOGP(debug, "loadFileToMemory {} ETag=[{}]", requestContext.path, requestContext.etag);

DataFormats/Detectors/CTP/include/DataFormatsCTP/Scalers.h

@@ -130,6 +130,19 @@ class CTPRunScalers
   /// same with absolute  timestamp (not orbit) as argument
   std::pair<double, double> getRateGivenT(double timestamp, int classindex, int type) const;
 
+  /// retrieves integral for class
+  std::array<uint64_t, 7> getIntegralForClass(int i) const
+  {
+    return {
+      mScalerRecordO2[0].scalers[i].classIndex,
+      mScalerRecordO2[mScalerRecordO2.size() - 1].scalers[i].lmBefore - mScalerRecordO2[0].scalers[i].lmBefore,
+      mScalerRecordO2[mScalerRecordO2.size() - 1].scalers[i].lmAfter - mScalerRecordO2[0].scalers[i].lmAfter,
+      mScalerRecordO2[mScalerRecordO2.size() - 1].scalers[i].l0Before - mScalerRecordO2[0].scalers[i].l0Before,
+      mScalerRecordO2[mScalerRecordO2.size() - 1].scalers[i].l0After - mScalerRecordO2[0].scalers[i].l0After,
+      mScalerRecordO2[mScalerRecordO2.size() - 1].scalers[i].l1Before - mScalerRecordO2[0].scalers[i].l1Before,
+      mScalerRecordO2[mScalerRecordO2.size() - 1].scalers[i].l1After - mScalerRecordO2[0].scalers[i].l1After,
+    };
+  }
   /// retrieves time boundaries of this scaler object from O2 scalers
   std::pair<unsigned long, unsigned long> getTimeLimit() const
   {

DataFormats/Detectors/TRD/include/DataFormatsTRD/RecoInputContainer.h

@@ -88,7 +88,7 @@ inline void RecoInputContainer::fillGPUIOPtr(o2::gpu::GPUTrackingInOutPointers*
   ptrs->nTRDTriggerRecords = mNTriggerRecords;
   ptrs->trdTriggerTimes = &(trdTriggerTimes[0]);
   ptrs->trdTrackletIdxFirst = &(trdTriggerIndices[0]);
-  ptrs->trdTrigRecMask = reinterpret_cast<const char*>(mTrigRecMask.data());
+  ptrs->trdTrigRecMask = reinterpret_cast<const uint8_t*>(mTrigRecMask.data());
   ptrs->nTRDTracklets = mNTracklets;
   ptrs->trdTracklets = reinterpret_cast<const o2::gpu::GPUTRDTrackletWord*>(mTracklets.data());
   ptrs->trdSpacePoints = reinterpret_cast<const o2::gpu::GPUTRDSpacePoint*>(mSpacePoints.data());

DataFormats/Parameters/CMakeLists.txt

@@ -14,7 +14,8 @@ o2_add_library(DataFormatsParameters
                        src/GRPLHCIFData.cxx
                        src/GRPECSObject.cxx
                        src/GRPMagField.cxx
-               PUBLIC_LINK_LIBRARIES FairRoot::Base O2::CommonConstants
+                       src/AggregatedRunInfo.cxx
+                PUBLIC_LINK_LIBRARIES FairRoot::Base O2::CommonConstants
                                      O2::CommonTypes O2::CCDB
                                      O2::DetectorsCommonDataFormats)
 
@@ -24,6 +25,7 @@ o2_target_root_dictionary(DataFormatsParameters
                                   include/DataFormatsParameters/GRPLHCIFData.h
                                   include/DataFormatsParameters/GRPECSObject.h
                                   include/DataFormatsParameters/GRPMagField.h
+                                  include/DataFormatsParameters/AggregatedRunInfo.h
                           LINKDEF src/ParametersDataLinkDef.h)
 
 o2_add_executable(simgrp-tool

DataFormats/Parameters/include/DataFormatsParameters/AggregatedRunInfo.h

@@ -0,0 +1,46 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file GRPECSObject.h
+/// \brief Header of the AggregatedRunInfo struct
+/// \author ruben.shahoyan@cern.ch sandro.wenzel@cern.ch
+
+#ifndef ALICEO2_DATA_AGGREGATEDRUNINFO_H_
+#define ALICEO2_DATA_AGGREGATEDRUNINFO_H_
+
+#include <cstdint>
+#include "CCDB/BasicCCDBManager.h"
+
+namespace o2::parameters
+{
+
+/// Composite struct where one may collect important global properties of data "runs"
+/// aggregated from various sources (GRPECS, RunInformation CCDB entries, etc.).
+/// Also offers the authoritative algorithms to collect these information for easy reuse
+/// across various algorithms (anchoredMC, analysis, ...)
+struct AggregatedRunInfo {
+  int runNumber;       // run number
+  int64_t sor;         // best known timestamp for the start of run
+  int64_t eor;         // best known timestamp for end of run
+  int64_t orbitsPerTF; // number of orbits per TF
+  int64_t orbitReset;  // timestamp of orbit reset before run
+  int64_t orbitSOR;    // orbit when run starts after orbit reset
+  int64_t orbitEOR;    // orbit when run ends after orbit reset
+
+  // we may have pointers to actual data source objects GRPECS, ...
+
+  // fills and returns AggregatedRunInfo for a given run number.
+  static AggregatedRunInfo buildAggregatedRunInfo(o2::ccdb::CCDBManagerInstance& ccdb, int runnumber);
+};
+
+} // namespace o2::parameters
+
+#endif

DataFormats/Parameters/src/AggregatedRunInfo.cxx

@@ -0,0 +1,86 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file AggregatedRunInfo.cxx
+/// \author sandro.wenzel@cern.ch
+
+#include "DataFormatsParameters/AggregatedRunInfo.h"
+#include "CCDB/BasicCCDBManager.h"
+#include "DataFormatsParameters/GRPECSObject.h"
+#include "CommonConstants/LHCConstants.h"
+#include "Framework/Logger.h"
+
+using namespace o2::parameters;
+
+o2::parameters::AggregatedRunInfo AggregatedRunInfo::buildAggregatedRunInfo(o2::ccdb::CCDBManagerInstance& ccdb, int runnumber)
+{
+  // TODO: could think about caching results per runnumber to
+  // avoid going to CCDB multiple times ---> but should be done inside the CCDBManagerInstance
+
+  // we calculate the first orbit of a run based on sor (start-of-run) and eor
+  // we obtain these by calling getRunDuration
+  auto [sor, eor] = ccdb.getRunDuration(runnumber);
+
+  // determine a good timestamp to query OrbitReset for this run
+  // --> the middle of the run is very appropriate and safer than just sor
+  auto run_mid_timestamp = sor + (eor - sor) / 2;
+
+  // query the time of the orbit reset (when orbit is defined to be 0)
+  auto ctpx = ccdb.getForTimeStamp<std::vector<Long64_t>>("CTP/Calib/OrbitReset", run_mid_timestamp);
+  int64_t tsOrbitReset = (*ctpx)[0]; // us
+
+  // get timeframe length from GRPECS
+  std::map<std::string, std::string> metadata;
+  metadata["runNumber"] = Form("%d", runnumber);
+  auto grpecs = ccdb.getSpecific<o2::parameters::GRPECSObject>("GLO/Config/GRPECS", run_mid_timestamp, metadata);
+  auto nOrbitsPerTF = grpecs->getNHBFPerTF();
+
+  // calculate SOR orbit
+  int64_t orbitSOR = (sor * 1000 - tsOrbitReset) / o2::constants::lhc::LHCOrbitMUS;
+  int64_t orbitEOR = (eor * 1000 - tsOrbitReset) / o2::constants::lhc::LHCOrbitMUS;
+
+  // adjust to the nearest TF edge to satisfy condition (orbitSOR % nOrbitsPerTF == 0)
+  orbitSOR = (orbitSOR / nOrbitsPerTF + 1) * nOrbitsPerTF; // +1 to choose the safe boundary ... towards run middle
+  orbitEOR = orbitEOR / nOrbitsPerTF * nOrbitsPerTF;
+
+  // fetch SOR directly from CTP entry on CCDB
+  bool oldFatalState = ccdb.getFatalWhenNull();
+  ccdb.setFatalWhenNull(false);
+  auto ctp_first_run_orbit = ccdb.getForTimeStamp<std::vector<int64_t>>("CTP/Calib/FirstRunOrbit", run_mid_timestamp);
+  ccdb.setFatalWhenNull(oldFatalState);
+  if (ctp_first_run_orbit && ctp_first_run_orbit->size() >= 3) {
+    // if we have CTP first run orbit available, we should use it
+
+    // int64_t creation_time = (*ctp_first_run_orbit)[0];
+    int64_t ctp_run_number = (*ctp_first_run_orbit)[1];
+    int64_t ctp_orbitSOR = (*ctp_first_run_orbit)[2];
+
+    if (ctp_run_number != runnumber) {
+      LOG(error) << "AggregatedRunInfo: run number inconsistency found (asked: " << runnumber << " vs CTP found: " << ctp_run_number << ")";
+    }
+
+    // overwrite orbitSOR
+    if (ctp_orbitSOR != orbitSOR) {
+      LOG(warn) << "The calculated orbitSOR " << orbitSOR << " differs from CTP orbitSOR " << ctp_orbitSOR;
+      // reasons for this is different unit of time storage in RunInformation (ms) and orbitReset (us), etc.
+
+      // so we need to adjust the SOR timings to be consistent
+      auto sor_new = (int64_t)((tsOrbitReset + ctp_orbitSOR * o2::constants::lhc::LHCOrbitMUS) / 1000.);
+      if (sor_new != sor) {
+        LOG(warn) << "Adjusting SOR from " << sor << " to " << sor_new;
+        sor = sor_new;
+      }
+    }
+    orbitSOR = ctp_orbitSOR;
+  }
+
+  return AggregatedRunInfo{runnumber, sor, eor, nOrbitsPerTF, tsOrbitReset, orbitSOR, orbitEOR};
+}

DataFormats/Parameters/src/ParametersDataLinkDef.h

@@ -30,5 +30,6 @@
 #pragma link C++ class o2::parameters::GRPMagField + ;
 #pragma link C++ class std::unordered_map < unsigned int, unsigned int> + ;
 #pragma link C++ class std::pair < unsigned long, std::string> + ;
+#pragma link C++ struct o2::parameters::AggregatedRunInfo + ;
 
 #endif

DataFormats/common/include/CommonDataFormat/IRFrame.h

@@ -28,9 +28,22 @@ namespace dataformats
 // We could just alias it to the bracket specialization, but this would create
 // problems with fwd.declaration
 struct IRFrame : public o2::math_utils::detail::Bracket<o2::InteractionRecord> {
+  static constexpr uint64_t LastIRFrame = uint64_t(0x1) << 63;
+
   using o2::math_utils::detail::Bracket<o2::InteractionRecord>::Bracket;
 
-  uint64_t info = 0;
+  uint64_t info = uint64_t(0);
+
+  void setLast(bool v = true)
+  {
+    if (v) {
+      info |= LastIRFrame;
+    } else {
+      v &= ~LastIRFrame;
+    }
+  }
+
+  bool isLast() const { return (info & LastIRFrame) != 0UL; }
 
   ClassDefNV(IRFrame, 2);
 };