gambit is hosted by Hepforge, IPPP Durham
GAMBIT  v1.5.0-2191-ga4742ac
a Global And Modular Bsm Inference Tool
hdf5reader.hpp
Go to the documentation of this file.
1 // GAMBIT: Global and Modular BSM Inference Tool
2 // *********************************************
23 
29 #include "gambit/Utils/cats.hpp"
31 
32 #include <boost/preprocessor/seq/for_each_i.hpp>
33 
34 #ifndef __hdf5_reader_hpp__
35 #define __hdf5_reader_hpp__
36 
37 namespace Gambit
38 {
39  namespace Printers
40  {
41 
45  static const std::size_t CHUNKLENGTH = 100;
46 
47  template<class T>
48  struct BuffPair
49  {
54  : data(d)
55  , isvalid(v)
56  {}
57  // Handy shortcut constructor
58  BuffPair(hid_t location_id, const std::string& name)
59  : data (location_id,name,true,'r')
60  , isvalid(location_id,name+"_isvalid",true,'r')
61  {}
62  // Default constructor, data uninitialised!
63  BuffPair() {}
64  };
65 
66  // Forward declaration
67  class SLHAcombo;
68 
73  template<class T>
75  {
76  private:
77  // Buffers local to a print function. Access whichever ones match the IDcode.
78  std::map<VBIDpair, BuffPair<T>> local_buffers;
79 
80  public:
83  {
84  }
85 
88  {
89  for(typename std::map<VBIDpair, BuffPair<T>>::iterator it=local_buffers.begin();
90  it!=local_buffers.end(); ++it)
91  {
92  it->second.data.closeDataSet();
93  it->second.isvalid.closeDataSet();
94  }
95  }
96 
99  BuffPair<T>& get_buffer(const int vID, const unsigned int i, const std::string& label, hid_t location_id);
100  };
101 
102  // A simple class to manage opening and closing a HDF5 file/group on construction and destruction
103  class HDF5File
104  {
105  public:
106  HDF5File(const std::string& file, const std::string& group);
107  ~HDF5File();
108  const hid_t file_id;
110  };
111 
112  class HDF5Reader : public BaseReader
113  {
114  public:
115  HDF5Reader(const Options& options);
116  ~HDF5Reader();
117 
119  virtual void reset(); // Reset 'read head' position to first entry
120  virtual ulong get_dataset_length(); // Get length of input dataset
121  virtual PPIDpair get_next_point(); // Get next rank/ptID pair in data file
122  virtual PPIDpair get_current_point(); // Get current rank/ptID pair in data file
123  virtual ulong get_current_index(); // Get a linear index which corresponds to the current rank/ptID pair in the iterative sense
124  virtual bool eoi(); // Check if 'current point' is past the end of the data file (and thus invalid!)
127  virtual std::size_t get_type(const std::string& label);
128  virtual std::set<std::string> get_all_labels(); // Get all dataset labels
129 
131  using BaseReader::_retrieve; // Tell compiler we are using some of the base class overloads of this on purpose.
132  #define DECLARE_RETRIEVE(r,data,i,elem) bool _retrieve(elem&, const std::string&, const uint, const ulong);
134  #ifndef SCANNER_STANDALONE
136  #endif
137  #undef DECLARE_RETRIEVE
138 
139  private:
140  // Location of HDF5 datasets to be read
141  const std::string file;
142  const std::string group;
143  HDF5File H5file; // contains file_id and location_id
144 
145  // Names of all datasets at the target location
146  const std::vector<std::string> all_datasets;
147 
148  // MPIrank and pointID dataset wrappers
153 
154  ulong current_dataset_index; // index in input dataset of the current read-head position
155  PPIDpair current_point; // PPID of the point at the current read-head position
156 
157  // PPIDpair and dataset index of the last retrieved data.
158  // Just to speed up "random access" retrieval of a lot of data from the same point
161 
162  // Search for the PPID supplied in the input data and return the index of the first match
163  ulong get_index_from_PPID(const PPIDpair);
164 
165  template<class T>
166  H5P_LocalReadBufferManager<T>& get_mybuffermanager();
167 
169  // Need one for every directly retrievable type, and a specialisation
170  // of 'get_mybuffermanager' to access it. But the latter have to be
171  // defined outside the class declaration, so they can be found below.
172  // Could create all these with a macro, but I am sick of macros so
173  // will just do it the "old-fashioned" way.
182 
184  template<class T>
185  bool _retrieve_template(T& out, const std::string& label, int aux_id, const uint rank, const ulong pointID)
186  {
187  // Retrieve the buffer manager for buffers with this type
188  auto& buffer_manager = get_mybuffermanager<T>();
189 
190  // Buffers are labelled by an IDcode, which in the printer case is a graph vertex.
191  // In the reader case I think we can safely re-use this system to assign IDs:
192  int IDcode = get_param_id(label);
193 
194  // Extract a buffer pair from the manager corresponding to this type + label
195  auto& selected_buffer = buffer_manager.get_buffer(IDcode, aux_id, label, H5file.location_id);
196 
197  // Determine the dataset index from which to extrat the input PPIDpair
198  ulong dset_index = get_index_from_PPID(PPIDpair(pointID,rank));
199 
200  // Extract data value
201  out = selected_buffer.data.get_entry(dset_index);
202 
203  // Extract data validity flag
204  return selected_buffer.isvalid.get_entry(dset_index);
205  }
206 
208  bool retrieve_and_add_to_SLHAea(SLHAstruct& out, bool& found, const std::string& spec_type, const std::string& entry, const SLHAcombo& item, const std::set<std::string>& all_dataset_labels, const uint rank, const ulong pointID);
209 
210  };
211 
213  template<class T>
214  BuffPair<T>& H5P_LocalReadBufferManager<T>::get_buffer(const int vertexID, const unsigned int aux_i, const std::string& label, hid_t location_id)
215  {
216  VBIDpair key;
217  key.vertexID = vertexID;
218  key.index = aux_i;
219 
220  typename std::map<VBIDpair, BuffPair<T>>::iterator it = local_buffers.find(key);
221 
222  if( it == local_buffers.end() )
223  {
224  error_if_key_exists(local_buffers, key, "local_buffers");
225  // No local buffer exists for this output stream yet, so make one
226 
227  // Create the new buffer objects
228  if(location_id<0)
229  {
230  std::ostringstream errmsg;
231  errmsg << "Failed to created HDF5 read buffer '"<<label<<"'! The suppied location_id does not point to a valid location in a HDF5 file!";
232  printer_error().raise(LOCAL_INFO, errmsg.str());
233  }
234 
235  local_buffers[key] = BuffPair<T>(location_id,label);
236 
237  // Get the new buffer back out of the map
238  it = local_buffers.find(key);
239  }
240 
241  if( it == local_buffers.end() )
242  {
243  std::ostringstream errmsg;
244  errmsg << "Error! Failed to retrieve newly created buffer (label="<<label<<") from local_buffers map! Key was: ("<<vertexID<<","<<aux_i<<")"<<std::endl;
245  printer_error().raise(LOCAL_INFO, errmsg.str());
246  }
247 
248  return it->second;
249  }
250 
252  #define DEFINE_BUFFMAN_GETTER(TYPE) \
253  template<> \
254  inline H5P_LocalReadBufferManager<TYPE>& \
255  HDF5Reader::get_mybuffermanager() \
256  { \
257  return CAT(hdf5_localbufferman_,TYPE); \
258  }
261  DEFINE_BUFFMAN_GETTER(long )
265  DEFINE_BUFFMAN_GETTER(float )
266  DEFINE_BUFFMAN_GETTER(double )
267  #undef DEFINE_BUFFMAN_GETTER
268 
269  // Register reader so it can be constructed via inifile instructions
270  // First argument is string label for inifile access, second is class from which to construct printer
271  LOAD_READER(hdf5, HDF5Reader)
272 
273  }
274 }
275 
276 #endif
bool _retrieve(T &, const std::string &label, const uint, const ulong)
Default _retrieve function.
EXPORT_SYMBOLS error & printer_error()
Printer errors.
#define LOCAL_INFO
Definition: local_info.hpp:34
H5P_LocalReadBufferManager< ulong > hdf5_localbufferman_ulong
Definition: hdf5reader.hpp:177
BOOST_PP_SEQ_FOR_EACH_I(GETTYPEID, _, PRINTABLE_TYPES) void printAllTypeIDs(void)
For debugging; print to stdout all the typeIDs for all types.
Definition: baseprinter.cpp:36
vertexID / sub-print index pair Identifies individual buffers (I call them VertexBuffer, but actually there can be more than one per vertex)
DataSetInterfaceScalar< int, CHUNKLENGTH > isvalid
Definition: hdf5reader.hpp:51
unsigned long long int ulonglong
~H5P_LocalReadBufferManager()
Destructor. Close all datasets.
Definition: hdf5reader.hpp:87
#define LOAD_READER(tag,...)
Definition: baseprinter.hpp:60
H5P_LocalReadBufferManager< int > hdf5_localbufferman_int
Buffer manager objects.
Definition: hdf5reader.hpp:174
const std::vector< std::string > all_datasets
Definition: hdf5reader.hpp:146
SLHAea::Coll SLHAstruct
Less confusing name for SLHAea container class.
EXPORT_SYMBOLS int get_param_id(const std::string &name, bool &is_new)
Consolidated &#39;get id&#39; function, for both main and aux.
bool _retrieve_template(T &out, const std::string &label, int aux_id, const uint rank, const ulong pointID)
"Master" templated retrieve function.
Definition: hdf5reader.hpp:185
DataSetInterfaceScalar< unsigned long, CHUNKLENGTH > pointIDs
Definition: hdf5reader.hpp:149
Nicer alias for SLHAea container class, and some convenient helper functions that add or retrieve the...
std::map< VBIDpair, BuffPair< T > > local_buffers
Definition: hdf5reader.hpp:78
dictionary item
Keeps track of vertex buffers local to a retrieve function Similar to the buffer manager for HDF5Prin...
Definition: hdf5reader.hpp:74
H5P_LocalReadBufferManager< float > hdf5_localbufferman_float
Definition: hdf5reader.hpp:180
DEFINE_BUFFMAN_GETTER(int) DEFINE_BUFFMAN_GETTER(uint) DEFINE_BUFFMAN_GETTER(long) DEFINE_BUFFMAN_GETTER(ulong) DEFINE_BUFFMAN_GETTER(longlong) DEFINE_BUFFMAN_GETTER(ulonglong) DEFINE_BUFFMAN_GETTER(float) DEFINE_BUFFMAN_GETTER(double) template< class BuffType > void H5P_LocalBufferManager< BuffType >
Templated H5P_LocalBufferManager member functions.
#define HDF5_RETRIEVABLE_TYPES
Definition: hdf5types.hpp:41
Declaration and definition of printer base class.
void error_if_key_exists(const std::map< T, U > &m, const T &key, const std::string &tag)
Helper function to check if a VertexBuffer key already exists in a map.
Definition: hdf5printer.hpp:74
BASE PRINTER CLASS.
H5P_LocalReadBufferManager< uint > hdf5_localbufferman_uint
Definition: hdf5reader.hpp:175
Tools for accessing printers.
unsigned long int ulong
BuffPair(DataSetInterfaceScalar< T, CHUNKLENGTH > &d, DataSetInterfaceScalar< int, CHUNKLENGTH > &v)
Definition: hdf5reader.hpp:52
#define DECLARE_RETRIEVE(r, data, i, elem)
Definition: hdf5reader.hpp:132
Declaration for class DataSetInterfaceScalar.
H5P_LocalReadBufferManager< ulonglong > hdf5_localbufferman_ulonglong
Definition: hdf5reader.hpp:179
DataSetInterfaceScalar< T, CHUNKLENGTH > data
Definition: hdf5reader.hpp:50
Derived dataset interface, with methods for writing scalar records (i.e.
H5P_LocalReadBufferManager< long > hdf5_localbufferman_long
Definition: hdf5reader.hpp:176
Sequence of all types printable by the HDF5 printer.
H5P_LocalReadBufferManager< longlong > hdf5_localbufferman_longlong
Definition: hdf5reader.hpp:178
DataSetInterfaceScalar< int, CHUNKLENGTH > pointIDs_isvalid
Definition: hdf5reader.hpp:150
H5P_LocalReadBufferManager< double > hdf5_localbufferman_double
Definition: hdf5reader.hpp:181
DataSetInterfaceScalar< int, CHUNKLENGTH > mpiranks
Definition: hdf5reader.hpp:151
A collection of tools for interacting with HDF5 databases.
BuffPair< T > & get_buffer(const int vID, const unsigned int i, const std::string &label, hid_t location_id)
Retrieve a buffer for an IDcode/auxilliary-index pair location_id used to access dataset if it has no...
Definition: hdf5reader.hpp:214
#define HDF5_BACKEND_TYPES
Definition: hdf5types.hpp:46
DataSetInterfaceScalar< int, CHUNKLENGTH > mpiranks_isvalid
Definition: hdf5reader.hpp:152
pointID / process number pair Used to identify a single parameter space point
TODO: see if we can use this one:
Definition: Analysis.hpp:33
A small wrapper object for &#39;options&#39; nodes.
long long int longlong
Concatenation macros.
BuffPair(hid_t location_id, const std::string &name)
Definition: hdf5reader.hpp:58