gambit is hosted by Hepforge, IPPP Durham
GAMBIT  v1.5.0-2191-ga4742ac
a Global And Modular Bsm Inference Tool
hdf5tools.cpp
Go to the documentation of this file.
1 // GAMBIT: Global and Modular BSM Inference Tool
2 // *********************************************
21 
24 #include "gambit/Logs/logger.hpp"
25 
26 #include <stdio.h>
27 #include <string.h>
28 #include <iostream>
29 
30 // Boost
31 #include <boost/preprocessor/seq/for_each.hpp>
32 
33 namespace Gambit {
34  namespace Printers {
35 
36  namespace HDF5 {
37 
39  // Sets some HDF5 properties to associate with open objects
40  // Here we set objects to be 'evicted' from the metadata cache
41  // when they are closed, which apparantly is not the default
42  // which leads to massive RAM usage if we don't set this.
44  {
45  hid_t fapl(H5Pcreate(H5P_FILE_ACCESS)); // Copy defaults
46  #ifdef HDF5_DEBUG
47  std::cout<<"HDF5 version:"<<H5_VERS_MAJOR<<"."<<H5_VERS_MINOR<<"."<<H5_VERS_RELEASE<<std::endl;
48  #endif
49  #if (H5_VERS_MAJOR > 1) || \
50  (H5_VERS_MAJOR == 1) && H5_VERS_MINOR > 10 || \
51  (H5_VERS_MAJOR == 1) && H5_VERS_MINOR == 10 && H5_VERS_RELEASE >= 1
52  hbool_t value = 1; // true?
53  // This function does not appear before v 1.10.1, however it is
54  // pretty crucial, at least in my version of HDF5, for keeping the
55  // metadata cache from consuming all my RAM. However, Anders commented
56  // it out with his older HDF5 version and still had no RAM problem.
57  // So it might be ok to just remove it for older versions.
58  // However, if you see RAM blowouts and your HDF5 version is old,
59  // then this is probably the reason.
60  H5Pset_evict_on_close(fapl, value); // Set evict_on_close = true
61  #ifdef HDF5_DEBUG
62  std::cout <<"GAMBIT fapl used!"<<std::endl; // Check that this code is built...
63  #endif
64  #endif
65 
66  return fapl;
67  }
68 
71 
73  #define SIMPLE_CALL(IDTYPE_OUT, FNAME, IDTYPE_IN, H5FUNCTION, VERB, OUTPUTNAME, INPUTNAME) \
74  IDTYPE_OUT FNAME(IDTYPE_IN id) \
75  { \
76  if(id < 0) \
77  { \
78  std::ostringstream errmsg; \
79  errmsg << "Failed to "<<VERB<<" "<<OUTPUTNAME<<" for HDF5 dataset! The supplied id does not point to a successfully opened "<<INPUTNAME<<"!"; \
80  printer_error().raise(LOCAL_INFO, errmsg.str()); \
81  } \
82  IDTYPE_OUT out_id = H5FUNCTION(id); \
83  if(out_id < 0) \
84  { \
85  std::ostringstream errmsg; \
86  errmsg << "Failed to "<<VERB<<" "<<OUTPUTNAME<<" for HDF5 dataset! See HDF5 error output for more details."; \
87  printer_error().raise(LOCAL_INFO, errmsg.str()); \
88  } \
89  return out_id; \
90  }
91 
93  {
94  if(id < 0)
95  {
96  std::ostringstream errmsg;
97  errmsg << "Failed to close HDF5 file with ID "<<id<<"! The supplied id does not point to a successfully opened file.";
98  printer_error().raise(LOCAL_INFO, errmsg.str());
99  }
100 
101  // Check for any open objects! These should all be closed before the file is closed for maximum safety
102  ssize_t count = H5Fget_obj_count(id, H5F_OBJ_ALL);
103  if(count > 1)
104  {
106  logger() << "Warning! "<<count<<" open objects detected when closing HDF5 file with ID "<<id<<"! Please check your code and ensure that all datasets, groups, selections, etc. are closed before closing the files they belong to."<<std::endl;
107  logger() << "Beginning analysis of open objects..."<<std::endl;
108  count = H5Fget_obj_count(id, H5F_OBJ_FILE);
109  if(count>1) logger() << " "<<count<<" H5F_OBJ_FILE detected (should only be 1, for the open file itself)"<<std::endl;
110  count = H5Fget_obj_count(id, H5F_OBJ_GROUP);
111  if(count>0) logger() << " "<<count<<" H5F_OBJ_GROUP detected"<<std::endl;
112  count = H5Fget_obj_count(id, H5F_OBJ_DATASET);
113  if(count>0) logger() << " "<<count<<" H5F_OBJ_DATASET detected"<<std::endl;
114  count = H5Fget_obj_count(id, H5F_OBJ_DATATYPE);
115  if(count>0) logger() << " "<<count<<" H5F_OBJ_DATATYPE detected"<<std::endl;
116  count = H5Fget_obj_count(id, H5F_OBJ_ATTR);
117  if(count>0) logger() << " "<<count<<" H5F_OBJ_ATTR detected"<<std::endl;
118  logger()<<EOM;
119  }
120 
121  hid_t out_id = H5Fclose(id);
122  if(out_id < 0)
123  {
124  std::ostringstream errmsg;
125  errmsg << "Failed to close HDF5 file with ID "<<id<<"! See HDF5 error output for more details.";
126  printer_error().raise(LOCAL_INFO, errmsg.str());
127  }
128  #ifdef HDF5_DEBUG
129  std::cout<<"Called H5Fclose on file with ID "<<id<<std::endl;
130  #endif
131  return out_id;
132  }
133 
134 
135  template<>
136  std::vector<bool> getChunk(const hid_t dset_id, std::size_t offset, std::size_t length)
137  {
138  // Buffer to receive data (and return from function)
139  std::vector<uint8_t> chunkdata(length);
140 
141  // Select hyperslab
142  std::pair<hid_t,hid_t> selection_ids = selectChunk(dset_id,offset,length);
143  hid_t memspace_id = selection_ids.first;
144  hid_t dspace_id = selection_ids.second;
145 
146  // Buffer to receive data
147  void* buffer = chunkdata.data(); // pointer to contiguous memory within the buffer vector
148 
149  // Get the data from the hyperslab.
150  hid_t hdftype_id = get_hdf5_data_type<bool>::type(); // It is assumed that you already know this is the right type for the dataset!
151  herr_t err_read = H5Dread(dset_id, hdftype_id, memspace_id, dspace_id, H5P_DEFAULT, buffer);
152 
153  if(err_read<0)
154  {
155  std::ostringstream errmsg;
156  errmsg << "Error retrieving chunk (offset="<<offset<<", length="<<length<<") from dataset in HDF5 file. H5Dread failed." << std::endl;
157  errmsg << " offset+length = "<< offset+length << std::endl;
158  printer_error().raise(LOCAL_INFO, errmsg.str());
159  }
160 
161  H5Sclose(dspace_id);
162  H5Sclose(memspace_id);
163 
164  std::vector<bool> chunkdata_bool;
165  for(auto it=chunkdata.begin(); it!=chunkdata.end(); ++it)
166  {
167  chunkdata_bool.push_back(*it);
168  }
169 
170  return chunkdata_bool;
171  }
172 
174  hid_t openFile(const std::string& fname, bool overwrite, const char access_type)
175  {
176  bool tmp;
177  return openFile(fname,overwrite,tmp,access_type);
178  }
179 
182  hid_t openFile(const std::string& fname, bool overwrite, bool& oldfile, const char access_type)
183  {
184  //Debug
185  //std::cerr<<"Attempting to open file "<<fname<<" in mode "<<access_type<<" (overwrite="<<overwrite<<")"<<std::endl;
186 
187  hid_t file_id; // file handle
188 
189  unsigned int atype=0;
190  switch(access_type)
191  {
192  case 'r':
193  atype = H5F_ACC_RDONLY;
194  break;
195  case 'w':
196  // We let 'w' mean read/write here
197  atype = H5F_ACC_RDWR;
198  break;
199  default:
200  std::ostringstream errmsg;
201  errmsg << "Unrecognised access mode requested while trying to open HDF5 file! Saw '"<<access_type<<"'; only 'r' (read-only) and 'w' (read/wrtie) are valid. File was ("<<fname<<")";
202  printer_error().raise(LOCAL_INFO, errmsg.str());
203  break;
204  }
205 
206  if(overwrite)
207  {
208  // DANGER! Deletes existing file
209  if( remove(fname.c_str()) != 0 )
210  {
211  // Error deleting file, but probably it just didn't exist to delete
212  logger()<<LogTags::utils<<LogTags::warn<<"Failed to delete file '"<<fname<<"'! Maybe it didn't exist in the first place."<<EOM;
213  }
214  else// else deleted file with no problem
215  {
216  logger()<<LogTags::utils<<LogTags::info<<"Deleted pre-existing file "<<fname<<" (because overwrite=true)"<<EOM;
217  }
218  }
219 
220  errorsOff();
221  file_id = H5Fopen(fname.c_str(), atype, H5P_GAMBIT);
222  errorsOn();
223  if(file_id < 0)
224  {
225  if(access_type=='w')
226  {
227  /* Ok maybe file doesn't exist yet, try creating it */
228  errorsOff();
229  file_id = H5Fcreate(fname.c_str(), H5F_ACC_EXCL, H5P_DEFAULT, H5P_GAMBIT);
230  errorsOn();
231  if(file_id < 0)
232  {
233  /* Still no good; error */
234  std::ostringstream errmsg;
235  errmsg << "Failed to open existing HDF5 file, then failed to create new one! ("<<fname<<"). The file may exist but be unreadable. You can check this by trying to inspect it with the 'h5ls' command line tool.";
236  printer_error().raise(LOCAL_INFO, errmsg.str());
237  }
238  else
239  {
240  /* successfully created new file */
241  oldfile = false;
242  }
243  }
244  else
245  {
246  // Doesn't make sense to create new file if we wanted read-only mode. Error.
247  std::ostringstream errmsg;
248  errmsg << "Failed to open existing HDF5 file, and did not create new one since read-only access was specified. ("<<fname<<")";
249  printer_error().raise(LOCAL_INFO, errmsg.str());
250  }
251  }
252  else
253  {
254  /* successfully opened existing file */
255  oldfile = true;
256  }
257 
258  // DEBUG
259  #ifdef HDF5_DEBUG
260  std::cout<<"Opened file "<<fname<<" in mode "<<access_type<<", and assigned it ID "<<file_id<<std::endl;
261  #endif
262 
263  /* Return the file handle */
264  return file_id;
265  }
266 
268  bool checkFileReadable(const std::string& fname, std::string& msg)
269  {
270  bool readable(false);
271 
272  errorsOff();
273  hid_t file_id = H5Fopen(fname.c_str(), H5F_ACC_RDONLY, H5P_GAMBIT);
274  errorsOn();
275  if(file_id < 0)
276  {
277  readable=false;
278  std::ostringstream errmsg;
279  errmsg<<"H5Fopen failed (tried to open '"<<fname<<"')";
280  msg = errmsg.str();
281  }
282  else
283  {
284  /* everything fine, close the file */
285  herr_t status = H5Fclose(file_id);
286  if(status<0)
287  {
288  std::ostringstream errmsg;
289  errmsg << "Failed to properly close HDF5 file after successfully checking that it was readable! ("<<fname<<")";
290  printer_error().raise(LOCAL_INFO, errmsg.str());
291  }
292  readable=true;
293  }
294  // DEBUG
295  #ifdef HDF5_DEBUG
296  std::cout<<"Checked that file "<<fname<<" was readable (had RDONLY access and ID "<<file_id<<")"<<std::endl;
297  #endif
298  return readable;
299  }
300 
302  bool checkGroupReadable(hid_t location, const std::string& groupname, std::string& msg)
303  {
304  hid_t group_id;
305  bool readable(false);
306 
307  errorsOff();
308  group_id = H5Gopen2(location, groupname.c_str(), H5P_DEFAULT);
309  errorsOn();
310  if(group_id < 0)
311  {
312  readable=false;
313  std::ostringstream errmsg;
314  errmsg<<"H5Gopen failed (tried to open '"<<groupname<<"' from location with id "<<location<<")";
315  msg = errmsg.str();
316  }
317  else
318  {
319  /* everything fine, close the group */
320  herr_t status = H5Gclose(group_id);
321  if(status<0)
322  {
323  std::ostringstream errmsg;
324  errmsg << "Failed to properly close HDF5 group after successfully checking that it was readable! ("<<groupname<<")";
325  printer_error().raise(LOCAL_INFO, errmsg.str());
326  }
327  readable=true;
328  }
329  return readable;
330  }
331 
332  template<class T>
333  std::pair<bool,std::size_t> _checkDatasetReadable_helper(hid_t dset_id, const std::string dset_name)
334  {
335  static const std::size_t CHUNK(1000);
336  std::vector<T> buffer(CHUNK);
337  bool fully_readable(true);
338  std::size_t largest_readable_index(0);
339 
340  // Get dataset length
341  hid_t dspace_id = getSpace(dset_id);
342  if(dspace_id<0)
343  {
344  fully_readable = false;
345  }
346  else
347  {
348  size_t dset_length(0);
349  bool length_error(false);
350  try
351  {
352  dset_length = getSimpleExtentNpoints(dspace_id);
353  }
354  catch(const Gambit::exception& e)
355  {
356  fully_readable = false;
357  length_error = true;
358  }
359  closeSpace(dspace_id);
360 
361  if(not length_error)
362  {
363  // Begin trying to read data
364  std::size_t Nchunks = dset_length / CHUNK;
365  std::size_t remainder = dset_length % CHUNK;
366  if(remainder!=0) Nchunks+=1;
367  std::size_t offset(0);
368  std::size_t length(0);
369  errorsOff();
370  for(std::size_t i=0; i<Nchunks; i++)
371  {
372  offset = i * CHUNK;
373  length = CHUNK;
374  if(remainder!=0 and (i+1)==Nchunks) length = remainder;
375  try
376  {
377  errorsOff();
378  buffer = getChunk<T>(dset_id, offset, length);
379  }
380  catch(const Gambit::exception& e)
381  {
382  fully_readable = false;
383  }
384  if(not fully_readable) break;
385  }
386  errorsOn();
387 
388  if(not fully_readable)
389  {
390  // Try to find highest readable index in the dataset
391  // We know it is somewhere in the last chunk we were reading.
392  // Could do a more efficient search, but we will just look
393  // sequentially from the beginning of the chunk
394 
395  errorsOff();
396  for(std::size_t j=offset; j<offset+length; j++)
397  {
398  try
399  {
400  std::vector<T> jbuffer = getChunk<T>(dset_id, j, 1);
401  largest_readable_index = j;
402  }
403  catch(const Gambit::exception& e)
404  {
405  break;
406  }
407  }
408  errorsOn();
409 
410  if(largest_readable_index==dset_length)
411  {
412  // Chunked read failed, but individual reads succeeded? Weird.
413  // Will have to abandon our efforts and make the user investigate
414  // manually
415  std::ostringstream err;
416  err<<"Dataset "<<dset_name<<" was determined to be partially unreadable (corrupted), however we were unable to determine the largest readable index. You will have to investigate the HDF5 file manually.";
417  printer_error().raise(LOCAL_INFO,err.str());
418  }
419  }
420  else
421  {
422  // Everything seems fine with this dataset
423  largest_readable_index = dset_length;
424  }
425  }
426  }
427  return std::make_pair(fully_readable,largest_readable_index);
428  }
429 
432  std::pair<bool,std::size_t> checkDatasetReadable(hid_t location, const std::string& dsetname)
433  {
434  std::pair<bool,std::size_t> readable_info(false,0);
435  hid_t dataset_id = openDataset(location, dsetname);
436  if(dataset_id<0)
437  {
438  //msg += "Failed to open dataset";
439  }
440  else
441  {
442  hid_t datatype_id = H5Dget_type(dataset_id);
443  if(datatype_id<0)
444  {
445  //msg += "Failed to obtain type of dataset";
446  }
447  else
448  {
449  // Need buffers of various types depending of type of dataset.
450  // Can achieve this with some macros and a templated helper function
451  #define RUN_TYPE_DEPENDENT_CHECK(r,data,elem) \
452  if( H5Tequal(datatype_id, get_hdf5_data_type<elem>::type()) )\
453  {\
454  readable_info = _checkDatasetReadable_helper<elem>(dataset_id,dsetname);\
455  }\
456  else
457  BOOST_PP_SEQ_FOR_EACH(RUN_TYPE_DEPENDENT_CHECK, _, H5_OUTPUT_TYPES)
458  #undef RUN_TYPE_DEPENDENT_CHECK
459  {
460  std::ostringstream err;
461  err << "Did not recognise retrieved HDF5 type for dataset '"<<dsetname<<"'! This may indicate a bug in the GAMBIT HDF5 tools library, please report it.";
462  printer_error().raise(LOCAL_INFO,err.str());
463  }
464  }
465  }
466  closeDataset(dataset_id);
467  return readable_info;
468  }
469 
471  hid_t createFile(const std::string& fname)
472  {
473  hid_t file_id = H5Fcreate(fname.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, H5P_GAMBIT);
474  if(file_id < 0)
475  {
476  /* Still no good; error */
477  std::ostringstream errmsg;
478  errmsg << "Failed to create HDF5 file '"<<fname<<"'!";
479  printer_error().raise(LOCAL_INFO, errmsg.str());
480  }
481  return file_id;
482  }
483 
485  // Argument "location" can be a handle for either a file or another group
486  hid_t createGroup(hid_t location, const std::string& name)
487  {
488  hid_t group_id;
489 
490  group_id = H5Gcreate2(location, name.c_str(), H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
491  if(group_id<0)
492  {
493  std::ostringstream errmsg;
494  errmsg << "Error creating HDF5 group '"<<name<<"'";
495  printer_error().raise(LOCAL_INFO, errmsg.str());
496  }
497  return group_id;
498  }
499 
500  // Modified minimally from https://github.com/gregreen/h5utils/blob/master/src/h5utils.cpp#L92
501  // Credit: Gregory Green 2012
502  /*
503  * Opens a group, creating it if it does not exist. Nonexistent parent groups are also
504  * created. This works similarly to the Unix/Linux command
505  * mkdir -p /parent/subgroup/group
506  * in that if /parent and /parent/subgroup do not exist, they will be created.
507  *
508  * If no accessmode has H5Utils::DONOTCREATE flag set, then returns NULL if group
509  * does not yet exist.
510  *
511  */
512  hid_t openGroup(hid_t file_id, const std::string& name, bool nocreate) //, int accessmode)
513  {
514  hid_t group_id;
515 
516  if(file_id < 0)
517  {
518  std::ostringstream errmsg;
519  errmsg << "Error opening HDF5 group '"<<name<<"'. The supplied file_id does not point to a successfully opened file!";
520  printer_error().raise(LOCAL_INFO, errmsg.str());
521  }
522 
523  // User does not want to create group
524  if(nocreate) //accessmode & H5Utils::DONOTCREATE)
525  {
526  group_id = H5Gopen2(file_id, name.c_str(), H5P_DEFAULT);
527  if(group_id<0)
528  {
529  std::ostringstream errmsg;
530  errmsg << "Error opening HDF5 group '"<<name<<"'. Group (probably) does not exist, and 'nocreate' flag is set to 'true', so we will not attempt to create one";
531  printer_error().raise(LOCAL_INFO, errmsg.str());
532  }
533  }
534  else
535  {
536  // Possibly create group and parent groups
537  std::stringstream ss(name);
538  std::stringstream path;
539  std::string gp_name;
540  while(std::getline(ss, gp_name, '/'))
541  {
542  path << "/" << gp_name;
543  errorsOff();
544  group_id = H5Gopen2(file_id, path.str().c_str(), H5P_DEFAULT);
545  errorsOn();
546  if(group_id<0)
547  {
548  /* doesn't exist; try to create it */
549  group_id = H5Gcreate2(file_id, path.str().c_str(), H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
550  if(group_id<0)
551  {
552  std::ostringstream errmsg;
553  errmsg << "Error while recursively creating/opening group '"<<name<<"'. Failed to create group '"<<path.str()<<"'";
554  printer_error().raise(LOCAL_INFO, errmsg.str());
555  }
556  }
557  herr_t err = H5Gclose(group_id);
558  if(err<0)
559  {
560  std::ostringstream errmsg;
561  errmsg << "Error closing group '"<<name<<"'!";
562  printer_error().raise(LOCAL_INFO, errmsg.str());
563  }
564  }
565  // Should exist now; open the group and return the handle
566  group_id = H5Gopen2(file_id, name.c_str(), H5P_DEFAULT);
567  if(group_id<0)
568  {
569  std::ostringstream errmsg;
570  errmsg << "Error opening HDF5 group '"<<name<<"' after recursive creation supposedly succeeded! There must be a bug in this routine, please fix.";
571  printer_error().raise(LOCAL_INFO, errmsg.str());
572  }
573  }
574  return group_id;
575  }
576 
577  // Iterator function for listing datasets in a group
578  herr_t group_ls(hid_t g_id, const char *name, const H5L_info_t* /*info*/, void *op_data)
579  {
580  #ifdef HDF5_DEBUG
581  //std::cout<<"group_ls: "<<name<<std::endl;
582  //std::cout<<info->type<<" "<<H5G_DATASET<<std::endl;
583  #endif
584  std::vector<std::string>* out = static_cast<std::vector<std::string>*>(op_data);
585  // Only add names that correspond to datasets
586  H5G_stat_t statbuf;
587  H5Gget_objinfo(g_id, name, false, &statbuf);
588  if(statbuf.type == H5G_DATASET) out->push_back(name);
589  return 0;
590  }
591 
593  std::vector<std::string> lsGroup(hid_t group_id)
594  {
595  if(group_id<0)
596  {
597  std::ostringstream errmsg;
598  errmsg << "Error inspecting HDF5 group. The supplied group_id does not point to an open group object!";
599  printer_error().raise(LOCAL_INFO, errmsg.str());
600  }
601 
602  std::vector<std::string> out;
603  herr_t err = H5Literate(group_id, H5_INDEX_NAME, H5_ITER_NATIVE, NULL, group_ls, &out);
604 
605  if(err<0)
606  {
607  std::ostringstream errmsg;
608  errmsg << "Error encountering while iterating through HDF5 group! See HDF5 error for more details (stderr).";
609  printer_error().raise(LOCAL_INFO, errmsg.str());
610  }
611 
612  return out;
613  }
614 
616  bool isDataSet(hid_t loc_id, const std::string& name)
617  {
618  H5O_info_t object_info;
619  herr_t err = H5Oget_info_by_name(loc_id, name.c_str(), &object_info, H5P_DEFAULT);
620  if(err<0)
621  {
622  std::ostringstream errmsg;
623  errmsg << "Attempt to check if object named '"<<name<<"' is a dataset failed! See HDF5 error for more details (stderr).";
624  printer_error().raise(LOCAL_INFO, errmsg.str());
625  }
626  return object_info.type == H5O_TYPE_DATASET;
627  }
628 
631  hid_t getH5DatasetType(hid_t group_id, const std::string& dset_name)
632  {
633  hid_t dataset_id = openDataset(group_id, dset_name);
634  if(dataset_id<0)
635  {
636  std::ostringstream errmsg;
637  errmsg << "Failed to open dataset '"<<dset_name<<"' while attempting to check its HDF5 data type! See stderr output for more details.";
638  printer_error().raise(LOCAL_INFO, errmsg.str());
639  }
640  hid_t type_id = H5Dget_type(dataset_id);
641  if(type_id<0)
642  {
643  std::ostringstream errmsg;
644  errmsg << "Failed to get HDF5 type of dataset '"<<dset_name<<"'. See stderr output for more details.";
645  printer_error().raise(LOCAL_INFO, errmsg.str());
646  }
647  closeDataset(dataset_id);
648  return type_id;
649  }
650 
652  SIMPLE_CALL(hid_t, closeType, hid_t, H5Tclose, "close", "type ID", "type ID")
653 
654 
655  SIMPLE_CALL(hid_t, closeGroup, hid_t, H5Gclose, "close", "group", "group")
656 
658  H5E_auto2_t old_func;
660 
661  // FIXME: This caused compile problems on LISA cluster (CW)
671  void errorsOff()
672  {
673  /* Save old error handler */
674  H5Eget_auto2(H5E_DEFAULT, &old_func, &old_client_data);
675 
676  /* Turn off error handling */
677  H5Eset_auto2(H5E_DEFAULT, NULL, NULL);
678  }
679 
681  void errorsOn()
682  {
683  /* Restore previous error handler */
684  H5Eset_auto2(H5E_DEFAULT, old_func, old_client_data);
685  }
686 
688 
690  // Set error_off=true to manually check for successful dataset opening.
691  hid_t openDataset(hid_t group_id, const std::string& name, bool error_off)
692  {
693  hid_t dset_id;
694 
695  if(group_id < 0)
696  {
697  std::ostringstream errmsg;
698  errmsg << "Error opening HDF5 dataset '"<<name<<"'. The supplied group_id in which the dataset should be located does not point to a successfully opened group!";
699  printer_error().raise(LOCAL_INFO, errmsg.str());
700  }
701 
702  dset_id = H5Dopen2(group_id, name.c_str(), H5P_DEFAULT);
703  if(dset_id<0 and not error_off)
704  {
705  std::ostringstream errmsg;
706  errmsg << "Error opening HDF5 dataset '"<<name<<"'. Dataset may not exist at the specified location.";
707  printer_error().raise(LOCAL_INFO, errmsg.str());
708  }
709  return dset_id;
710  }
711 
713  SIMPLE_CALL(hid_t, closeDataset, hid_t, H5Dclose, "close", "dataset", "dataset")
714 
715 
716  SIMPLE_CALL(hid_t, getSpace, hid_t, H5Dget_space, "get", "dataspace", "dataset")
717  SIMPLE_CALL(hid_t, closeSpace, hid_t, H5Sclose, "close", "dataspace", "dataspace")
718 
721 
723  std::string getName(hid_t dset_id)
724  {
725  size_t len = H5Iget_name(dset_id,NULL,0);
726  char buffer[len];
727  H5Iget_name(dset_id,buffer,len+1);
728  std::string n = buffer;
729  return n;
730  }
731 
733  std::pair<hid_t,hid_t> selectChunk(const hid_t dset_id, std::size_t offset, std::size_t length)
734  {
735  // Open dataspace
736  hid_t dspace_id = getSpace(dset_id);
737 
738  // Make sure that the requested chunk lies within the dataset extents
739  size_t dset_length = getSimpleExtentNpoints(dspace_id);
740 
741  if(offset + length > dset_length)
742  {
743  std::ostringstream errmsg;
744  errmsg << "Error selecting chunk from dataset in HDF5 file. Tried to select a hyperslab which extends beyond the dataset extents:" << std::endl;
745  errmsg << " offset = " << offset << std::endl;
746  errmsg << " offset+length = " << length << std::endl;
747  errmsg << " dset_length = "<< dset_length << std::endl;
748  printer_error().raise(LOCAL_INFO, errmsg.str());
749  }
750 
751  // Select a hyperslab.
752  static const size_t DSETRANK(1); // assuming 1D dataset
753  hsize_t offsets[DSETRANK];
754  offsets[0] = offset;
755  hsize_t selection_dims[DSETRANK]; // Set same as output chunks, but may have a different length
756  selection_dims[0] = length; // Adjust chunk length to input specification
757 
758  herr_t err_hs = H5Sselect_hyperslab(dspace_id, H5S_SELECT_SET, offsets, NULL, selection_dims, NULL);
759  if(err_hs<0)
760  {
761  std::ostringstream errmsg;
762  errmsg << "Error selecting chunk from dataset (offset="<<offset<<", length="<<selection_dims[0]<<") in HDF5 file. H5Sselect_hyperslab failed." << std::endl;
763  printer_error().raise(LOCAL_INFO, errmsg.str());
764  }
765 
766  // Define memory space
767  hid_t memspace_id = H5Screate_simple(DSETRANK, selection_dims, NULL);
768 
769  #ifdef HDF5_DEBUG
770  std::cout << "Debug variables:" << std::endl
771  << " dsetdims()[0] = " << this->dsetdims()[0] << std::endl
772  << " offsets[0] = " << offsets[0] << std::endl
773  << " CHUNKLENGTH = " << CHUNKLENGTH << std::endl
774  << " selection_dims[0] = " << selection_dims[0] << std::endl;
775  #endif
776 
777  return std::make_pair(memspace_id, dspace_id); // Be sure to close these identifiers after using them!
778  }
779 
781 
782  // Match fixed integers to HDF5 types
784  {
785  #define ELSEIF(r,data,elem) \
786  else if(H5Tequal(h5type,get_hdf5_data_type<elem>::type())) \
787  { \
788  out = h5v2_type<elem>(); \
789  }
790 
791  int out;
792  if(h5type==-1)
793  {
794  std::ostringstream errmsg;
795  errmsg<<"No fixed ID assigned for this type! (h5type = "<<h5type<<")!";
796  printer_error().raise(LOCAL_INFO, errmsg.str());
797  }
798  BOOST_PP_SEQ_FOR_EACH(ELSEIF, _, H5_OUTPUT_TYPES)
799  #undef ELSEIF
800  else
801  {
802  std::ostringstream errmsg;
803  errmsg<<"Unrecognised HDF5 type (h5type = "<<h5type<<")!";
804  printer_error().raise(LOCAL_INFO, errmsg.str());
805  }
806  return out;
807  }
808 
809  // Query whether type integer indicates general 'float' or 'int'
810  bool is_float_type(int inttype)
811  {
812  bool out(false);
813  switch(inttype)
814  {
815  case h5v2_type<int >():
817  case h5v2_type<long >():
819  case h5v2_type<long long >():
821  out = false;
822  break;
823  case h5v2_type<float >():
824  case h5v2_type<double >():
825  out = true;
826  break;
827  }
828  return out;
829  }
830 
831  }
832 
834  void printAllH5Types(void)
835  {
836  std::cout << "Types known to get_hdf5_data_type<T>::type() function:" << std::endl;
837  #define PRINTTYPEID(r,data,elem) \
838  std::cout << " Type: " << STRINGIFY(elem) << ", H5 type code: " << get_hdf5_data_type<elem>::type() << std::endl;
839  BOOST_PP_SEQ_FOR_EACH(PRINTTYPEID, _, H5_OUTPUT_TYPES)
840  #undef PRINTTYPEID
841  }
842 
843  }
844 }
845 
846 
847 
#define PRINTTYPEID(r, data, elem)
herr_t group_ls(hid_t g_id, const char *name, const H5L_info_t *, void *op_data)
Definition: hdf5tools.cpp:578
std::pair< bool, std::size_t > checkDatasetReadable(hid_t location, const std::string &dsetname)
Check if a dataset exists and can be read from fully (Reads through entire dataset to make sure! May ...
Definition: hdf5tools.cpp:432
hid_t openDataset(hid_t dset_id, const std::string &name, bool error_off=false)
Dataset and dataspace manipulation.
Definition: hdf5tools.cpp:691
constexpr int h5v2_type< long long >()
Definition: hdf5tools.hpp:283
dataset H5Sget_simple_extent_npoints
Definition: hdf5tools.cpp:720
#define RUN_TYPE_DEPENDENT_CHECK(r, data, elem)
EXPORT_SYMBOLS error & printer_error()
Printer errors.
std::pair< bool, std::size_t > _checkDatasetReadable_helper(hid_t dset_id, const std::string dset_name)
Definition: hdf5tools.cpp:333
constexpr int h5v2_type< int >()
Definition: hdf5tools.hpp:279
constexpr int h5v2_type< unsigned long >()
Definition: hdf5tools.hpp:282
LOCAL_INFO macro.
#define LOCAL_INFO
Definition: local_info.hpp:34
const hid_t H5P_GAMBIT(create_GAMBIT_fapl())
Const global for the GAMBIT fapl.
STL namespace.
dataset getSimpleExtentNpoints
Definition: hdf5tools.cpp:720
hid_t openFile(const std::string &fname, bool overwrite, bool &oldfile, const char access_type='r')
File and group manipulation.
Definition: hdf5tools.cpp:182
Logging access header for GAMBIT.
hid_t create_GAMBIT_fapl()
GAMBIT default file access property list.
Definition: hdf5tools.cpp:43
hid_t closeType(hid_t type_id)
Release datatype identifier.
constexpr int h5v2_type< unsigned int >()
Definition: hdf5tools.hpp:280
group H5E_auto2_t old_func
global error variables (handler)
Definition: hdf5tools.cpp:655
void errorsOff()
Silence error report (e.g. while probing for file existence)
Definition: hdf5tools.cpp:671
void printAllH5Types(void)
DEBUG: print to stdout all HDF5 type IDs.
Definition: hdf5tools.cpp:834
int inttype_from_h5type(hid_t h5type)
Definition: hdf5tools.cpp:783
hid_t createFile(const std::string &fname)
Create hdf5 file (always overwrite existing files)
Definition: hdf5tools.cpp:471
bool is_float_type(int inttype)
Definition: hdf5tools.cpp:810
hid_t createGroup(hid_t location, const std::string &name)
Create a group inside the specified location.
Definition: hdf5tools.cpp:486
const Logging::endofmessage EOM
Explicit const instance of the end of message struct in Gambit namespace.
Definition: logger.hpp:100
EXPORT_SYMBOLS Logging::LogMaster & logger()
Function to retrieve a reference to the Gambit global log object.
Definition: logger.cpp:95
hid_t closeFile(hid_t file)
Close hdf5 file.
Definition: hdf5tools.cpp:92
hid_t getH5DatasetType(hid_t group_id, const std::string &dset_name)
Get type of an object in a group.
Definition: hdf5tools.cpp:631
std::vector< T > getChunk(const hid_t dset_id, std::size_t offset, std::size_t length)
Retrieve a chunk of data from a simple dataset NOTE! Doesn&#39;t work for T=bool! Have a custom specialis...
Definition: hdf5tools.hpp:179
bool checkFileReadable(const std::string &fname, std::string &msg)
Check if hdf5 file exists and can be opened in read/write mode.
Definition: hdf5tools.cpp:268
std::vector< std::string > lsGroup(hid_t group_id)
List object names in a group.
Definition: hdf5tools.cpp:593
#define ELSEIF(r, data, elem)
constexpr int h5v2_type< double >()
Definition: hdf5tools.hpp:286
constexpr int h5v2_type< long >()
Definition: hdf5tools.hpp:281
std::string getName(hid_t dset_id)
Get name of dataset.
Definition: hdf5tools.cpp:723
hid_t openGroup(hid_t file_id, const std::string &name, bool nocreate=false)
Definition: hdf5tools.cpp:512
constexpr int h5v2_type< float >()
Definition: hdf5tools.hpp:285
hid_t closeSpace(hid_t space_id)
Close dataspace.
bool checkGroupReadable(hid_t location, const std::string &groupname, std::string &msg)
Check if a group exists and can be accessed.
Definition: hdf5tools.cpp:302
bool isDataSet(hid_t group_id, const std::string &name)
Check if an object in a group is a dataset.
Definition: hdf5tools.cpp:616
A collection of tools for interacting with HDF5 databases.
Base template is left undefined in order to raise a compile error if specialisation doesn&#39;t exist...
Definition: hdf5tools.hpp:77
constexpr int h5v2_type< unsigned long long >()
Definition: hdf5tools.hpp:284
TODO: see if we can use this one:
Definition: Analysis.hpp:33
void errorsOn()
Restore error report.
Definition: hdf5tools.cpp:681
hid_t closeDataset(hid_t dset_id)
Close dataset.
std::pair< hid_t, hid_t > selectChunk(const hid_t dset_id, std::size_t offset, std::size_t length)
Select a simple hyperslab in a 1D dataset.
Definition: hdf5tools.cpp:733
#define H5_OUTPUT_TYPES
Definition: hdf5tools.hpp:244
SIMPLE_CALL(hid_t, closeType, hid_t, H5Tclose, "close", "type ID", "type ID") SIMPLE_CALL(hid_t
Close hdf5 type ID.