Implement persistence for ProcessBlock products

Author: wddgit

DQMServices/FwkIO/plugins/DQMRootSource.cc

@@ -325,7 +325,7 @@ class DQMRootSource : public edm::PuttableSourceBase, DQMTTreeIO {
 
   edm::InputSource::ItemType getNextItemType() override;
 
-  std::unique_ptr<edm::FileBlock> readFile_() override;
+  std::shared_ptr<edm::FileBlock> readFile_() override;
   std::shared_ptr<edm::RunAuxiliary> readRunAuxiliary_() override;
   std::shared_ptr<edm::LuminosityBlockAuxiliary> readLuminosityBlockAuxiliary_() override;
   void readRun_(edm::RunPrincipal& rpCache) override;
@@ -465,7 +465,7 @@ DQMRootSource::~DQMRootSource() {
 edm::InputSource::ItemType DQMRootSource::getNextItemType() { return m_nextItemType; }
 
 // We will read the metadata of all files and fill m_fileMetadatas vector
-std::unique_ptr<edm::FileBlock> DQMRootSource::readFile_() {
+std::shared_ptr<edm::FileBlock> DQMRootSource::readFile_() {
   const int numFiles = m_catalog.fileNames(0).size();
   m_openFiles.reserve(numFiles);
 
@@ -601,7 +601,7 @@ std::unique_ptr<edm::FileBlock> DQMRootSource::readFile_() {
     m_nextItemType = edm::InputSource::IsRun;
 
   // We have to return something but not sure why
-  return std::make_unique<edm::FileBlock>();
+  return std::make_shared<edm::FileBlock>();
 }
 
 std::shared_ptr<edm::RunAuxiliary> DQMRootSource::readRunAuxiliary_() {

DataFormats/Provenance/interface/BranchType.h

@@ -9,9 +9,12 @@
 #include "FWCore/Utilities/interface/BranchType.h"
 
 namespace edm {
+
   std::string const& BranchTypeToString(BranchType const& branchType);
 
+  constexpr unsigned int numberOfRunLumiEventProductTrees = 3;
   std::string const& BranchTypeToProductTreeName(BranchType const& branchType);
+  std::string BranchTypeToProductTreeName(BranchType const& branchType, std::string const& processName);
 
   std::string const& BranchTypeToMetaDataTreeName(BranchType const& branchType);
 
@@ -49,6 +52,7 @@ namespace edm {
     // Other branches on Events Tree
     std::string const& eventSelectionsBranchName();
     std::string const& branchListIndexesBranchName();
+    std::string const& eventToProcessBlockIndexesBranchName();
 
     //------------------------------------------------------------------
     //------------------------------------------------------------------
@@ -70,6 +74,7 @@ namespace edm {
     std::string const& fileIndexBranchName();  // backward compatibility
     std::string const& indexIntoFileBranchName();
     std::string const& mergeableRunProductMetadataBranchName();
+    std::string const& processBlockHelperBranchName();
 
     // Event History Tree // backward compatibility
     std::string const& eventHistoryTreeName();  // backward compatibility

DataFormats/Provenance/interface/EventToProcessBlockIndexes.h

@@ -0,0 +1,24 @@
+#ifndef DataFormats_Provenance_EventToProcessBlockIndexes_h
+#define DataFormats_Provenance_EventToProcessBlockIndexes_h
+
+/**
+
+\author W. David Dagenhart, created 5 January, 2021
+
+*/
+
+namespace edm {
+
+  class EventToProcessBlockIndexes {
+  public:
+    EventToProcessBlockIndexes() {}
+
+    unsigned int index() const { return index_; }
+    void setIndex(unsigned int value) { index_ = value; }
+
+  private:
+    unsigned int index_ = 0;
+  };
+
+}  // namespace edm
+#endif

DataFormats/Provenance/interface/ProvenanceFwd.h

@@ -12,6 +12,7 @@ namespace edm {
   class ProductProvenance;
   class EventAuxiliary;
   class EventID;
+  class EventToProcessBlockIndexes;
   class LuminosityBlockAuxiliary;
   class LuminosityBlockID;
   class ModuleDescription;
@@ -24,6 +25,7 @@ namespace edm {
   class RunAuxiliary;
   class RunID;
   class StableProvenance;
+  class StoredProcessBlockHelper;
   class Timestamp;
   class ProductProvenanceLookup;
 }  // namespace edm

DataFormats/Provenance/interface/StoredProcessBlockHelper.h

@@ -0,0 +1,50 @@
+#ifndef DataFormats_Provenance_StoredProcessBlockHelper_h
+#define DataFormats_Provenance_StoredProcessBlockHelper_h
+
+/** \class edm::StoredProcessBlockHelper
+
+This contains the information from the ProcessBlockHelper
+that is persistent. The processBlockCacheIndices_ data
+member is a flattened  version of the vector of vectors
+in the ProcessBlockHelper. It is flattened mainly for
+I/O performance reasons. This is intended to be directly
+used only by the IOPool code responsible for reading
+and writing the persistent files. Everything else should
+interact with the ProcessBlockHelper.
+
+\author W. David Dagenhart, created 1 Oct, 2020
+
+*/
+
+#include <string>
+#include <vector>
+
+namespace edm {
+
+  class StoredProcessBlockHelper {
+  public:
+    // This constructor exists for ROOT I/O
+    StoredProcessBlockHelper();
+
+    StoredProcessBlockHelper(std::vector<std::string> const& processesWithProcessBlockProducts);
+
+    std::vector<std::string> const& processesWithProcessBlockProducts() const {
+      return processesWithProcessBlockProducts_;
+    }
+
+    std::vector<std::string>& processesWithProcessBlockProducts() { return processesWithProcessBlockProducts_; }
+
+    std::vector<unsigned int> const& processBlockCacheIndices() const { return processBlockCacheIndices_; }
+
+    std::vector<unsigned int>& processBlockCacheIndices() { return processBlockCacheIndices_; }
+
+    static constexpr unsigned int invalidCacheIndex() { return 0xffffffff; }
+    static constexpr unsigned int invalidProcessIndex() { return 0xffffffff; }
+
+  private:
+    std::vector<std::string> processesWithProcessBlockProducts_;
+
+    std::vector<unsigned int> processBlockCacheIndices_;
+  };
+}  // namespace edm
+#endif

DataFormats/Provenance/src/BranchType.cc

@@ -78,11 +78,13 @@ namespace edm {
     std::string const fileIndex = "FileIndex";
     std::string const indexIntoFile = "IndexIntoFile";
     std::string const mergeableRunProductMetadata = "MergeableRunProductMetadata";
+    std::string const processBlockHelper = "ProcessBlockHelper";
     std::string const eventHistory = "EventHistory";
     std::string const eventBranchMapper = "EventBranchMapper";
 
     std::string const eventSelections = "EventSelections";
     std::string const branchListIndexes = "BranchListIndexes";
+    std::string const eventToProcessBlockIndexes = "EventToProcessBlockIndexes";
 
     std::string const parameterSetsTree = "ParameterSets";
     std::string const idToParameterSetBlobsBranch = "IdToParameterSetsBlobs";
@@ -95,6 +97,11 @@ namespace edm {
     return treeNames[branchType];
   }
 
+  std::string BranchTypeToProductTreeName(BranchType const& branchType, std::string const& processName) {
+    assert(branchType == InProcess);
+    return branchTypeNames[InProcess] + "s" + processName;
+  }
+
   std::string const& BranchTypeToMetaDataTreeName(BranchType const& branchType) {
     assert(branchType < eventLumiRunSize);
     return metaTreeNames[branchType];
@@ -193,6 +200,9 @@ namespace edm {
     // Branch on MetaData Tree
     std::string const& mergeableRunProductMetadataBranchName() { return mergeableRunProductMetadata; }
 
+    // Branch on MetaData Tree
+    std::string const& processBlockHelperBranchName() { return processBlockHelper; }
+
     // Branch on Event History Tree
     std::string const& eventHistoryBranchName() { return eventHistory; }
 
@@ -201,6 +211,8 @@ namespace edm {
 
     std::string const& branchListIndexesBranchName() { return branchListIndexes; }
 
+    std::string const& eventToProcessBlockIndexesBranchName() { return eventToProcessBlockIndexes; }
+
     std::string const& parameterSetsTreeName() { return parameterSetsTree; }
     // Branch on ParameterSets Tree
     std::string const& idToParameterSetBlobsBranchName() { return idToParameterSetBlobsBranch; }

DataFormats/Provenance/src/ProductRegistry.cc

@@ -273,7 +273,8 @@ namespace edm {
         }
         ++i;
       } else if (i == e || (j != s && j->first < i->first)) {
-        if (j->second.present() && branchesMustMatch == BranchDescription::Strict) {
+        if (j->second.present() &&
+            (branchesMustMatch == BranchDescription::Strict || j->second.branchType() == InProcess)) {
           differences << "Branch '" << j->second.branchName() << "' is in previous files\n";
           differences << "    but not in file '" << fileName << "'.\n";
         }

DataFormats/Provenance/src/StoredProcessBlockHelper.cc

@@ -0,0 +1,10 @@
+#include "DataFormats/Provenance/interface/StoredProcessBlockHelper.h"
+
+namespace edm {
+
+  StoredProcessBlockHelper::StoredProcessBlockHelper() {}
+
+  StoredProcessBlockHelper::StoredProcessBlockHelper(std::vector<std::string> const& processesWithProcessBlockProducts)
+      : processesWithProcessBlockProducts_(processesWithProcessBlockProducts) {}
+
+}  // namespace edm

DataFormats/Provenance/src/classes.h

@@ -5,6 +5,7 @@
 #include "DataFormats/Provenance/interface/ElementID.h"
 #include "DataFormats/Provenance/interface/EventAuxiliary.h"
 #include "DataFormats/Provenance/interface/EventID.h"
+#include "DataFormats/Provenance/interface/EventToProcessBlockIndexes.h"
 #include "DataFormats/Provenance/interface/FileFormatVersion.h"
 #include "DataFormats/Provenance/interface/FileID.h"
 #include "DataFormats/Provenance/interface/FileIndex.h"
@@ -27,6 +28,7 @@
 #include "DataFormats/Provenance/interface/ProductID.h"
 #include "DataFormats/Provenance/interface/ProductProvenance.h"
 #include "DataFormats/Provenance/interface/StoredMergeableRunProductMetadata.h"
+#include "DataFormats/Provenance/interface/StoredProcessBlockHelper.h"
 #include "DataFormats/Provenance/interface/StoredProductProvenance.h"
 #include "DataFormats/Provenance/interface/ProductRegistry.h"
 #include "DataFormats/Provenance/interface/RunAuxiliary.h"

DataFormats/Provenance/src/classes_def.xml

@@ -147,6 +147,9 @@
  <class name="edm::StoredMergeableRunProductMetadata" ClassVersion="3">
   <version ClassVersion="3" checksum="2293482595"/>
  </class>
+ <class name="edm::StoredProcessBlockHelper" ClassVersion="3">
+  <version ClassVersion="3" checksum="3443065706"/>
+ </class>
  <class name="edm::ProcessHistoryID"/>
  <class name="std::set<edm::ProcessHistoryID >"/>
  <class name="std::vector<edm::ProcessHistory>"/>
@@ -217,6 +220,10 @@
  <class name="edm::ThinnedAssociationsHelper" ClassVersion="10">
   <version ClassVersion="10" checksum="3444973106"/>
  </class>
+ <class name="edm::EventToProcessBlockIndexes" ClassVersion="3">
+  <version ClassVersion="3" checksum="1281389788"/>
+ </class>
+
  <ioread sourceClass="edm::ProductRegistry" targetClass="edm::ProductRegistry" version="[1-]" source="" target="transient_">
  <![CDATA[
 	newObj->initializeTransients();