root_serialization/HDFBatchEventsOutputer.h at master · hep-cce2/root_serialization · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
#if !defined(HDFBatchEventsOutputer_h)
#define HDFBatchEventsOutputer_h


#include <vector>
#include <string>
#include <cstdint>
#include <fstream>


#include "OutputerBase.h"
#include "EventIdentifier.h"
#include "SerializeStrategy.h"
#include "DataProductRetriever.h"
#include "pds_writer.h"

#include "SerialTaskQueue.h"

#include "HDFCxx.h"


namespace cce::tf {
  class HDFBatchEventsOutputer : public OutputerBase {
    public:
    enum class CompressionChoice {
      kNone,
        kEvents,
        kBatch,
        kBoth
    };

    HDFBatchEventsOutputer(std::string const& iFileName, unsigned int iNLanes, int iChunkSize, pds::Compression iCompression, int iCompressionLevel, CompressionChoice iChoice, pds::Serialization iSerialization, uint32_t iBatchSize);
    HDFBatchEventsOutputer(HDFBatchEventsOutputer&&) = default;
    HDFBatchEventsOutputer(HDFBatchEventsOutputer const&) = default;

  void setupForLane(unsigned int iLaneIndex, std::vector<DataProductRetriever> const& iDPs) final;

  void productReadyAsync(unsigned int iLaneIndex, DataProductRetriever const& iDataProduct, TaskHolder iCallback) const final;
  bool usesProductReadyAsync() const final {return true;}

  void outputAsync(unsigned int iLaneIndex, EventIdentifier const& iEventID, TaskHolder iCallback) const final;

  void printSummary() const final;

 private:

  void finishBatchAsync(unsigned int iBatchIndex, TaskHolder iCallback);

  void output(std::vector<EventIdentifier> iEventID, std::vector<char> iBuffer, std::vector<uint32_t> iOffset);
  void writeFileHeader(SerializeStrategy const& iSerializers);
  std::pair<std::vector<uint32_t>,std::vector<char>> writeDataProductsToOutputBuffer(SerializeStrategy const& iSerializers) const;

private:
  hdf5::File file_;
  hdf5::Group group_;
  mutable SerialTaskQueue queue_;
  int chunkSize_;
  mutable std::vector<SerializeStrategy> serializers_;

  //This is used as a circular buffer of length nLanes but only entries being used exist
  using EventInfo = std::tuple<EventIdentifier, std::vector<uint32_t>, std::vector<char>>;
  mutable std::vector<std::atomic<std::vector<EventInfo>*>> eventBatches_;
  mutable std::vector<std::atomic<uint32_t>> waitingEventsInBatch_;

  mutable std::atomic<uint64_t> presentEventEntry_;

  uint32_t batchSize_;
  bool firstEvent_ = true;
  pds::Compression compression_;
  int compressionLevel_;
  CompressionChoice compressionChoice_;
  pds::Serialization serialization_;
  mutable std::chrono::microseconds serialTime_;
  mutable std::atomic<std::chrono::microseconds::rep> parallelTime_;
  };
}
#endif