gambit is hosted by Hepforge, IPPP Durham
GAMBIT  v1.5.0-2191-ga4742ac
a Global And Modular Bsm Inference Tool
hdf5tools.hpp
Go to the documentation of this file.
1 // GAMBIT: Global and Modular BSM Inference Tool
2 // *********************************************
25 
26 #ifndef __hdf5tools_hpp__
27 #define __hdf5tools_hpp__
28 
29 // Standard library
30 #include <cstdint>
31 #include <memory>
32 #include <sstream>
33 #include <iostream>
34 #include <algorithm>
35 
36 // GAMBIT
39 
40 // HDF5
41 #include <hdf5.h>
42 
43 // Boost
44 #include <boost/utility/enable_if.hpp>
45 
47 #define SPECIALISE_HDF5_DATA_TYPE_IF_NEEDED(TYPEDEFD_TYPE, RETURN_HDF5_TYPE) \
48  template<typename T> \
49  struct get_hdf5_data_type<T, typename boost::enable_if_c< std::is_same<T, TYPEDEFD_TYPE>::value && \
50  !std::is_same<char, TYPEDEFD_TYPE>::value && \
51  !std::is_same<short, TYPEDEFD_TYPE>::value && \
52  !std::is_same<int, TYPEDEFD_TYPE>::value && \
53  !std::is_same<long, TYPEDEFD_TYPE>::value && \
54  !std::is_same<long long, TYPEDEFD_TYPE>::value && \
55  !std::is_same<unsigned char, TYPEDEFD_TYPE>::value && \
56  !std::is_same<unsigned short, TYPEDEFD_TYPE>::value && \
57  !std::is_same<unsigned int, TYPEDEFD_TYPE>::value && \
58  !std::is_same<unsigned long, TYPEDEFD_TYPE>::value && \
59  !std::is_same<unsigned long long, TYPEDEFD_TYPE>::value && \
60  !std::is_same<float, TYPEDEFD_TYPE>::value && \
61  !std::is_same<double, TYPEDEFD_TYPE>::value && \
62  !std::is_same<long double, TYPEDEFD_TYPE>::value && \
63  !std::is_same<bool, TYPEDEFD_TYPE>::value>::type > \
64  { \
65  static hid_t type() { return RETURN_HDF5_TYPE; } \
66  }; \
67 
68 
69 namespace Gambit
70 {
71 
72  namespace Printers
73  {
76  template<typename T, typename Enable=void>
78 
79  namespace HDF5
80  {
81 
83 
85  // If overwrite=true then any existing file will be deleted and replaced. USE CAREFULLY!!!
86  // third argument "oldfile" is used to report whether an existing file was opened (true if yes)
87  hid_t openFile(const std::string& fname, bool overwrite, bool& oldfile, const char access_type='r');
88  hid_t openFile(const std::string& fname, bool overwrite=false, const char access_type='r');
89 
91  hid_t closeFile(hid_t file);
92 
94  bool checkFileReadable(const std::string& fname, std::string& msg);
96  inline bool checkFileReadable(const std::string& fname)
97  { std::string garbage; return checkFileReadable(fname, garbage); }
98 
100  bool checkGroupReadable(hid_t location, const std::string& groupname, std::string& msg);
102  inline bool checkGroupReadable(hid_t location, const std::string& groupname)
103  { std::string garbage; return checkGroupReadable(location, groupname, garbage); }
104 
107  std::pair<bool,std::size_t> checkDatasetReadable(hid_t location, const std::string& dsetname);
108 
110  hid_t createFile(const std::string& fname);
111 
113  // Argument "location" can be a pointer to either a file or another group
114  hid_t createGroup(hid_t location, const std::string& name);
115 
117  void errorsOff();
118 
120  void errorsOn();
121 
122  // Modified minimally from https://github.com/gregreen/h5utils/blob/master/src/h5utils.cpp#L92
123  // Credit: Gregory Green 2012
124  /*
125  * Opens a group, creating it if it does not exist. Nonexistent parent groups are also
126  * created. This works similarly to the Unix/Linux command
127  * mkdir -p /parent/subgroup/group
128  * in that if /parent and /parent/subgroup do not exist, they will be created.
129  *
130  * If no accessmode has H5Utils::DONOTCREATE flag set, then returns NULL if group
131  * does not yet exist.
132  *
133  */
134  hid_t openGroup(hid_t file_id, const std::string& name, bool nocreate=false);
135 
138 
140  std::vector<std::string> lsGroup(hid_t group_id);
141 
143  hid_t getH5DatasetType(hid_t group_id, const std::string& dset_name);
144 
146  hid_t closeType(hid_t type_id);
147 
149 
151 
153  hid_t openDataset(hid_t dset_id, const std::string& name, bool error_off=false);
154 
156  hid_t closeDataset(hid_t dset_id);
157 
159  hid_t getSpace(hid_t dset_id);
160 
162  hid_t closeSpace(hid_t space_id);
163 
165  hssize_t getSimpleExtentNpoints(hid_t dset_id);
166 
168  std::string getName(hid_t dset_id);
169 
171  std::pair<hid_t,hid_t> selectChunk(const hid_t dset_id, std::size_t offset, std::size_t length);
172 
174  bool isDataSet(hid_t group_id, const std::string& name);
175 
178  template<class T>
179  std::vector<T> getChunk(const hid_t dset_id, std::size_t offset, std::size_t length)
180  {
181  // Buffer to receive data (and return from function)
182  std::vector<T> chunkdata(length);
183 
184  // Select hyperslab
185  std::pair<hid_t,hid_t> selection_ids = selectChunk(dset_id,offset,length);
186  hid_t memspace_id = selection_ids.first;
187  hid_t dspace_id = selection_ids.second;
188 
189  // Buffer to receive data
190  void* buffer = chunkdata.data(); // pointer to contiguous memory within the buffer vector
191 
192  // Get the data from the hyperslab.
193  hid_t hdftype_id = get_hdf5_data_type<T>::type(); // It is assumed that you already know this is the right type for the dataset!
194  herr_t err_read = H5Dread(dset_id, hdftype_id, memspace_id, dspace_id, H5P_DEFAULT, buffer);
195 
196  if(err_read<0)
197  {
198  std::ostringstream errmsg;
199  errmsg << "Error retrieving chunk (offset="<<offset<<", length="<<length<<") from dataset in HDF5 file. H5Dread failed." << std::endl;
200  errmsg << " offset+length = "<< offset+length << std::endl;
201  printer_error().raise(LOCAL_INFO, errmsg.str());
202  }
203 
204  H5Sclose(dspace_id);
205  H5Sclose(memspace_id);
206 
207  return chunkdata;
208  }
209 
210  // Bool version to deal with bool weirdness
211  template<>
212  std::vector<bool> getChunk(const hid_t dset_id, std::size_t offset, std::size_t length);
213 
214  // Match fixed integers to HDF5 types
215  int inttype_from_h5type(hid_t h5type);
216 
217  // Query whether type integer indicates general 'float' or 'int'
218  bool is_float_type(int inttype);
219 
221  }
222 
225  template<> struct get_hdf5_data_type<char> { static hid_t type() { return H5T_NATIVE_CHAR ; } };
226  template<> struct get_hdf5_data_type<short> { static hid_t type() { return H5T_NATIVE_SHORT ; } };
227  template<> struct get_hdf5_data_type<int> { static hid_t type() { return H5T_NATIVE_INT ; } };
228  template<> struct get_hdf5_data_type<long> { static hid_t type() { return H5T_NATIVE_LONG ; } };
229  template<> struct get_hdf5_data_type<long long> { static hid_t type() { return H5T_NATIVE_LLONG ; } };
230  template<> struct get_hdf5_data_type<unsigned char> { static hid_t type() { return H5T_NATIVE_UCHAR ; } };
231  template<> struct get_hdf5_data_type<unsigned short> { static hid_t type() { return H5T_NATIVE_USHORT ; } };
232  template<> struct get_hdf5_data_type<unsigned int> { static hid_t type() { return H5T_NATIVE_UINT ; } };
233  template<> struct get_hdf5_data_type<unsigned long> { static hid_t type() { return H5T_NATIVE_ULONG ; } };
234  template<> struct get_hdf5_data_type<unsigned long long>{ static hid_t type() { return H5T_NATIVE_ULLONG ; } };
235  template<> struct get_hdf5_data_type<float> { static hid_t type() { return H5T_NATIVE_FLOAT ; } };
236  template<> struct get_hdf5_data_type<double> { static hid_t type() { return H5T_NATIVE_DOUBLE ; } };
237  template<> struct get_hdf5_data_type<long double> { static hid_t type() { return H5T_NATIVE_LDOUBLE; } };
238  template<> struct get_hdf5_data_type<bool> { static hid_t type() { return H5T_NATIVE_UINT8 ; } };
239  // Bools are a bit trickier because C has no built-in boolean type (until recently; anyway
240  // the HDF5 libraries were written in C before this existed). We also want something that
241  // will be recognised as a bool by h5py. For now we just use an unsigned int.
242 
243  // Macro sequence for iterating over all allowed output types
244  #define H5_OUTPUT_TYPES \
245  (char) \
246  (short) \
247  (int) \
248  (long) \
249  (long long) \
250  (unsigned char) \
251  (unsigned short) \
252  (unsigned int) \
253  (unsigned long) \
254  (unsigned long long) \
255  (float) \
256  (double) \
257  (long double) \
258  (bool)
259 
261  void printAllH5Types(void);
262 
264 
267  SPECIALISE_HDF5_DATA_TYPE_IF_NEEDED(int8_t, H5T_NATIVE_INT8)
268  SPECIALISE_HDF5_DATA_TYPE_IF_NEEDED(uint8_t, H5T_NATIVE_UINT8)
269  SPECIALISE_HDF5_DATA_TYPE_IF_NEEDED(int16_t, H5T_NATIVE_INT16)
270  SPECIALISE_HDF5_DATA_TYPE_IF_NEEDED(uint16_t, H5T_NATIVE_UINT16)
271  SPECIALISE_HDF5_DATA_TYPE_IF_NEEDED(int32_t, H5T_NATIVE_INT32)
272  SPECIALISE_HDF5_DATA_TYPE_IF_NEEDED(uint32_t, H5T_NATIVE_UINT32)
273  SPECIALISE_HDF5_DATA_TYPE_IF_NEEDED(int64_t, H5T_NATIVE_INT64)
274  SPECIALISE_HDF5_DATA_TYPE_IF_NEEDED(uint64_t, H5T_NATIVE_UINT64)
276 
277  // Template function to match datatypes to fixed integers
278  template<class T> constexpr int h5v2_type() {return -1;}
279  template<> constexpr int h5v2_type<int >() {return 0;}
280  template<> constexpr int h5v2_type<unsigned int >() {return 1;}
281  template<> constexpr int h5v2_type<long >() {return 2;}
282  template<> constexpr int h5v2_type<unsigned long >() {return 3;}
283  template<> constexpr int h5v2_type<long long >() {return 4;}
284  template<> constexpr int h5v2_type<unsigned long long>() {return 5;}
285  template<> constexpr int h5v2_type<float >() {return 6;}
286  template<> constexpr int h5v2_type<double >() {return 7;}
287 
289 
290  }
291 }
292 
293 #endif
294 
std::pair< bool, std::size_t > checkDatasetReadable(hid_t location, const std::string &dsetname)
Check if a dataset exists and can be read from fully (Reads through entire dataset to make sure! May ...
Definition: hdf5tools.cpp:432
DecayTable::Entry DecayTable::Entry DecayTable::Entry DecayTable::Entry DecayTable::Entry DecayTable::Entry DecayTable::Entry DecayTable::Entry DecayTable::Entry DecayTable::Entry DecayTable::Entry DecayTable::Entry DecayTable::Entry DecayTable::Entry DecayTable::Entry DecayTable::Entry DecayTable::Entry DecayTable::Entry DecayTable::Entry DecayTable::Entry DecayTable::Entry DecayTable::Entry DecayTable::Entry DecayTable::Entry DecayTable::Entry DecayTable::Entry DecayTable::Entry DecayTable::Entry DecayTable::Entry DecayTable::Entry DecayTable::Entry DecayTable::Entry DecayTable::Entry DecayTable::Entry DecayTable::Entry DecayTable::Entry DecayTable::Entry DecayTable::Entry DecayTable::Entry DecayTable::Entry DecayTable::Entry DecayTable::Entry DecayTable::Entry DecayTable::Entry DecayTable::Entry DecayTable::Entry DecayTable::Entry DecayTable::Entry DecayTable::Entry DecayTable::Entry DecayTable::Entry DecayTable::Entry DecayTable::Entry DecayTable::Entry DecayTable::Entry DecayTable::Entry DecayTable::Entry double
hid_t openDataset(hid_t dset_id, const std::string &name, bool error_off=false)
Dataset and dataspace manipulation.
Definition: hdf5tools.cpp:691
constexpr int h5v2_type< long long >()
Definition: hdf5tools.hpp:283
EXPORT_SYMBOLS error & printer_error()
Printer errors.
constexpr int h5v2_type< int >()
Definition: hdf5tools.hpp:279
constexpr int h5v2_type< unsigned long >()
Definition: hdf5tools.hpp:282
#define LOCAL_INFO
Definition: local_info.hpp:34
dataset getSimpleExtentNpoints
Definition: hdf5tools.cpp:720
hid_t openFile(const std::string &fname, bool overwrite, bool &oldfile, const char access_type='r')
File and group manipulation.
Definition: hdf5tools.cpp:182
hid_t closeType(hid_t type_id)
Release datatype identifier.
constexpr int h5v2_type< unsigned int >()
Definition: hdf5tools.hpp:280
void errorsOff()
Silence error report (e.g. while probing for file existence)
Definition: hdf5tools.cpp:671
void printAllH5Types(void)
DEBUG: print to stdout all HDF5 type IDs.
Definition: hdf5tools.cpp:834
int inttype_from_h5type(hid_t h5type)
Definition: hdf5tools.cpp:783
hid_t createFile(const std::string &fname)
Create hdf5 file (always overwrite existing files)
Definition: hdf5tools.cpp:471
bool is_float_type(int inttype)
Definition: hdf5tools.cpp:810
hid_t createGroup(hid_t location, const std::string &name)
Create a group inside the specified location.
Definition: hdf5tools.cpp:486
hid_t closeFile(hid_t file)
Close hdf5 file.
Definition: hdf5tools.cpp:92
hid_t getH5DatasetType(hid_t group_id, const std::string &dset_name)
Get type of an object in a group.
Definition: hdf5tools.cpp:631
std::vector< T > getChunk(const hid_t dset_id, std::size_t offset, std::size_t length)
Retrieve a chunk of data from a simple dataset NOTE! Doesn&#39;t work for T=bool! Have a custom specialis...
Definition: hdf5tools.hpp:179
bool checkFileReadable(const std::string &fname, std::string &msg)
Check if hdf5 file exists and can be opened in read/write mode.
Definition: hdf5tools.cpp:268
DS5_MSPCTM DS_INTDOF int
std::vector< std::string > lsGroup(hid_t group_id)
List object names in a group.
Definition: hdf5tools.cpp:593
A simple C++ wrapper for the MPI C bindings.
constexpr int h5v2_type< double >()
Definition: hdf5tools.hpp:286
constexpr int h5v2_type< long >()
Definition: hdf5tools.hpp:281
Exception objects required for standalone compilation.
std::string getName(hid_t dset_id)
Get name of dataset.
Definition: hdf5tools.cpp:723
hid_t openGroup(hid_t file_id, const std::string &name, bool nocreate=false)
Definition: hdf5tools.cpp:512
H5T_NATIVE_UINT8 H5T_NATIVE_UINT16 H5T_NATIVE_UINT32 H5T_NATIVE_UINT64 constexpr int h5v2_type()
Definition: hdf5tools.hpp:278
constexpr int h5v2_type< float >()
Definition: hdf5tools.hpp:285
hid_t closeSpace(hid_t space_id)
Close dataspace.
bool checkGroupReadable(hid_t location, const std::string &groupname, std::string &msg)
Check if a group exists and can be accessed.
Definition: hdf5tools.cpp:302
bool isDataSet(hid_t group_id, const std::string &name)
Check if an object in a group is a dataset.
Definition: hdf5tools.cpp:616
Base template is left undefined in order to raise a compile error if specialisation doesn&#39;t exist...
Definition: hdf5tools.hpp:77
constexpr int h5v2_type< unsigned long long >()
Definition: hdf5tools.hpp:284
TODO: see if we can use this one:
Definition: Analysis.hpp:33
void errorsOn()
Restore error report.
Definition: hdf5tools.cpp:681
hid_t closeDataset(hid_t dset_id)
Close dataset.
SPECIALISE_HDF5_DATA_TYPE_IF_NEEDED(int8_t, H5T_NATIVE_INT8) SPECIALISE_HDF5_DATA_TYPE_IF_NEEDED(uint8_t
Typedef&#39;d types; enabled only where they differ from the true types.
std::pair< hid_t, hid_t > selectChunk(const hid_t dset_id, std::size_t offset, std::size_t length)
Select a simple hyperslab in a 1D dataset.
Definition: hdf5tools.cpp:733