root_serialization/HDFOutputer.h at master · Dr15Jones/root_serialization · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
#if !defined(HDFOutputer_h)
#define HDFOutputer_h


#include <vector>
#include <string>
#include <cstdint>
#include <fstream>


#include "OutputerBase.h"
#include "EventIdentifier.h"
#include "SerializerWrapper.h"
#include "DataProductRetriever.h"

#include "SerialTaskQueue.h"

#include "HDFCxx.h"

using product_t = std::vector<char>;

namespace cce::tf {
  class HDFOutputer : public OutputerBase {
    public:
    HDFOutputer(std::string const& iFileName, unsigned int iNLanes, int iBatchSize, int iChunkSize);
    HDFOutputer(HDFOutputer&&) = default;
    HDFOutputer(HDFOutputer const&) = default;
    ~HDFOutputer();

  void setupForLane(unsigned int iLaneIndex, std::vector<DataProductRetriever> const& iDPs) final;

  void productReadyAsync(unsigned int iLaneIndex, DataProductRetriever const& iDataProduct, TaskHolder iCallback) const final;
  bool usesProductReadyAsync() const final {return true;}

  void outputAsync(unsigned int iLaneIndex, EventIdentifier const& iEventID, TaskHolder iCallback) const final;

  void printSummary() const final;

 private:

  void output(EventIdentifier const& iEventID, std::vector<SerializerWrapper> const& iSerializers);
  void writeFileHeader(EventIdentifier const& iEventID, std::vector<SerializerWrapper> const& iSerializers);

  std::vector<std::vector<char>> writeDataProductsToOutputBuffer(std::vector<SerializerWrapper> const& iSerializers) const;
std::pair<product_t, std::vector<size_t>> get_prods_and_sizes(std::vector<product_t> & input,int prod_index,int stride);
private:
 void writeBatch();

  hdf5::File file_;
  mutable SerialTaskQueue queue_;
  int chunkSize_;
  int maxBatchSize_;
  std::vector<std::pair<std::string, uint32_t>> dataProductIndices_;
  mutable std::vector<std::vector<SerializerWrapper>> serializers_;
  bool firstTime_ = true;

  std::vector<product_t> products_;
  std::vector<size_t> offsets_ = decltype(offsets_)(1000, 0);
  mutable int batch_ = 0;
  std::vector<int> events_;

  mutable std::chrono::microseconds serialTime_;
  mutable std::atomic<std::chrono::microseconds::rep> parallelTime_;
  };
}
#endif