gambit is hosted by Hepforge, IPPP Durham
GAMBIT  v1.5.0-252-gf9a3f78
a Global And Modular Bsm Inference Tool
Gambit::Printers::HDF5 Namespace Reference

Classes

struct  copy_hdf5
 
class  hdf5_stuff
 
struct  ra_copy_hdf5
 
struct  read_hdf5
 

Functions

template<typename T >
type_ret ()
 
template<class U , typename... T>
void Enter_HDF5 (hid_t dataset, T &... params)
 
void combine_hdf5_files (const std::string output_file, const std::string &base_file_name, const std::string &group, const size_t num, const bool resume, const bool cleanup, const bool skip, const std::vector< std::string > input_files=std::vector< std::string >())
 
std::unordered_map< PPIDpair, unsigned long long, PPIDHash, PPIDEqualget_RA_write_hash (hid_t group_id, std::unordered_set< PPIDpair, PPIDHash, PPIDEqual > &left_to_match)
 Helper function to create output hash map for RA points note: left_to_match points will be erased as we go, and are passed by reference, so will be erased in calling context also. More...
 
std::pair< std::vector< std::string >, std::vector< size_t > > find_temporary_files (const std::string &finalfile)
 Search for temporary files to be combined. More...
 
std::pair< std::vector< std::string >, std::vector< size_t > > find_temporary_files (const std::string &finalfile, size_t &max_i)
 Search for temporary files to be combined. More...
 
hsize_t getGroupNum (hid_t group_id)
 
hid_t getType (hid_t dataset)
 
std::vector< std::string > get_dset_names (hid_t group_id)
 
herr_t op_func (hid_t loc_id, const char *name_in, const H5L_info_t *, void *operator_data)
 
herr_t op_func_aux (hid_t loc_id, const char *name_in, const H5L_info_t *, void *operator_data)
 
void setup_hdf5_points (hid_t new_group, hid_t type, hid_t type2, unsigned long long size_tot, const std::string &name)
 
std::vector< std::string > getGroups (std::string groups)
 
hid_t create_GAMBIT_fapl ()
 GAMBIT default file access property list. More...
 
const hid_t H5P_GAMBIT (create_GAMBIT_fapl())
 Const global for the GAMBIT fapl. More...
 
template<class T >
std::pair< bool, std::size_t > _checkDatasetReadable_helper (hid_t dset_id, const std::string dset_name)
 
herr_t group_ls (hid_t g_id, const char *name, const H5L_info_t *, void *op_data)
 
 SIMPLE_CALL (hid_t, closeType, hid_t, H5Tclose, "close", "type ID", "type ID") SIMPLE_CALL(hid_t
 Close hdf5 type ID. More...
 
hid_t openFile (const std::string &fname, bool overwrite, bool &oldfile, const char access_type='r')
 File and group manipulation. More...
 
hid_t openFile (const std::string &fname, bool overwrite, const char access_type)
 Create or open hdf5 file (ignoring feedback regarding whether file already existed) More...
 
hid_t closeFile (hid_t file)
 Close hdf5 file. More...
 
bool checkFileReadable (const std::string &fname, std::string &msg)
 Check if hdf5 file exists and can be opened in read/write mode. More...
 
bool checkFileReadable (const std::string &fname)
 Thin wrapper for the above to discard failure message. More...
 
bool checkGroupReadable (hid_t location, const std::string &groupname, std::string &msg)
 Check if a group exists and can be accessed. More...
 
bool checkGroupReadable (hid_t location, const std::string &groupname)
 Thin wrapper for the above to discard failure message. More...
 
std::pair< bool, std::size_t > checkDatasetReadable (hid_t location, const std::string &dsetname)
 Check if a dataset exists and can be read from fully (Reads through entire dataset to make sure! May take some time) More...
 
hid_t createFile (const std::string &fname)
 Create hdf5 file (always overwrite existing files) More...
 
hid_t createGroup (hid_t location, const std::string &name)
 Create a group inside the specified location. More...
 
void errorsOff ()
 Silence error report (e.g. while probing for file existence) More...
 
void errorsOn ()
 Restore error report. More...
 
hid_t openGroup (hid_t file_id, const std::string &name, bool nocreate=false)
 
hid_t closeGroup (hid_t group)
 Close group. More...
 
std::vector< std::string > lsGroup (hid_t group_id)
 List object names in a group. More...
 
hid_t getH5DatasetType (hid_t group_id, const std::string &dset_name)
 Get type of an object in a group. More...
 
hid_t closeType (hid_t type_id)
 Release datatype identifier. More...
 
hid_t openDataset (hid_t dset_id, const std::string &name, bool error_off=false)
 Dataset and dataspace manipulation. More...
 
hid_t closeDataset (hid_t dset_id)
 Close dataset. More...
 
hid_t getSpace (hid_t dset_id)
 Get dataspace. More...
 
hid_t closeSpace (hid_t space_id)
 Close dataspace. More...
 
hssize_t getSimpleExtentNpoints (hid_t dset_id)
 Get simple dataspace extent. More...
 
std::string getName (hid_t dset_id)
 Get name of dataset. More...
 
std::pair< hid_t, hid_tselectChunk (const hid_t dset_id, std::size_t offset, std::size_t length)
 Select a simple hyperslab in a 1D dataset. More...
 
bool isDataSet (hid_t group_id, const std::string &name)
 Check if an object in a group is a dataset. More...
 
template<class T >
std::vector< T > getChunk (const hid_t dset_id, std::size_t offset, std::size_t length)
 Retrieve a chunk of data from a simple dataset NOTE! Doesn't work for T=bool! Have a custom specialisation in the source file for that. More...
 
template<>
std::vector< bool > getChunk (const hid_t dset_id, std::size_t offset, std::size_t length)
 
int inttype_from_h5type (hid_t h5type)
 
bool is_float_type (int inttype)
 

Variables

 closeGroup
 
 H5Gclose
 
 close
 
 group
 
group H5E_auto2_t old_func
 global error variables (handler) More...
 
voidold_client_data
 
 hid_t
 
 getSpace
 
 H5Dget_space
 
 get
 
 dataspace
 
dataset getSimpleExtentNpoints
 
dataset H5Sget_simple_extent_npoints
 
dataset simple_extent_npoints
 
 SIMPLE_CALL (hid_t, closeDataset, hid_t, H5Dclose, "close", "dataset", "dataset") SIMPLE_CALL(hid_t
 Close dataset. More...
 
dataset SIMPLE_CALL (hid_t, closeSpace, hid_t, H5Sclose, "close", "dataspace", "dataspace") SIMPLE_CALL(hssize_t
 Get simple dataspace extent (number of points); input dataspace, output data extent (size) More...
 

Function Documentation

◆ _checkDatasetReadable_helper()

template<class T >
std::pair<bool,std::size_t> Gambit::Printers::HDF5::_checkDatasetReadable_helper ( hid_t  dset_id,
const std::string  dset_name 
)

Definition at line 333 of file hdf5tools.cpp.

References closeSpace(), combine_hdf5::dset_length, Gambit::LogTags::err, errorsOff(), errorsOn(), getSimpleExtentNpoints, getSpace, hid_t, LOCAL_INFO, and Gambit::Printers::printer_error().

334  {
335  static const std::size_t CHUNK(1000);
336  std::vector<T> buffer(CHUNK);
337  bool fully_readable(true);
338  std::size_t largest_readable_index(0);
339 
340  // Get dataset length
341  hid_t dspace_id = getSpace(dset_id);
342  if(dspace_id<0)
343  {
344  fully_readable = false;
345  }
346  else
347  {
348  size_t dset_length(0);
349  bool length_error(false);
350  try
351  {
352  dset_length = getSimpleExtentNpoints(dspace_id);
353  }
354  catch(const Gambit::exception& e)
355  {
356  fully_readable = false;
357  length_error = true;
358  }
359  closeSpace(dspace_id);
360 
361  if(not length_error)
362  {
363  // Begin trying to read data
364  std::size_t Nchunks = dset_length / CHUNK;
365  std::size_t remainder = dset_length % CHUNK;
366  if(remainder!=0) Nchunks+=1;
367  std::size_t offset(0);
368  std::size_t length(0);
369  errorsOff();
370  for(std::size_t i=0; i<Nchunks; i++)
371  {
372  offset = i * CHUNK;
373  length = CHUNK;
374  if(remainder!=0 and (i+1)==Nchunks) length = remainder;
375  try
376  {
377  errorsOff();
378  buffer = getChunk<T>(dset_id, offset, length);
379  }
380  catch(const Gambit::exception& e)
381  {
382  fully_readable = false;
383  }
384  if(not fully_readable) break;
385  }
386  errorsOn();
387 
388  if(not fully_readable)
389  {
390  // Try to find highest readable index in the dataset
391  // We know it is somewhere in the last chunk we were reading.
392  // Could do a more efficient search, but we will just look
393  // sequentially from the beginning of the chunk
394 
395  errorsOff();
396  for(std::size_t j=offset; j<offset+length; j++)
397  {
398  try
399  {
400  std::vector<T> jbuffer = getChunk<T>(dset_id, j, 1);
401  largest_readable_index = j;
402  }
403  catch(const Gambit::exception& e)
404  {
405  break;
406  }
407  }
408  errorsOn();
409 
410  if(largest_readable_index==dset_length)
411  {
412  // Chunked read failed, but individual reads succeeded? Weird.
413  // Will have to abandon our efforts and make the user investigate
414  // manually
415  std::ostringstream err;
416  err<<"Dataset "<<dset_name<<" was determined to be partially unreadable (corrupted), however we were unable to determine the largest readable index. You will have to investigate the HDF5 file manually.";
417  printer_error().raise(LOCAL_INFO,err.str());
418  }
419  }
420  else
421  {
422  // Everything seems fine with this dataset
423  largest_readable_index = dset_length;
424  }
425  }
426  }
427  return std::make_pair(fully_readable,largest_readable_index);
428  }
EXPORT_SYMBOLS error & printer_error()
Printer errors.
#define LOCAL_INFO
Definition: local_info.hpp:34
dataset getSimpleExtentNpoints
Definition: hdf5tools.cpp:720
void errorsOff()
Silence error report (e.g. while probing for file existence)
Definition: hdf5tools.cpp:671
hid_t closeSpace(hid_t space_id)
Close dataspace.
void errorsOn()
Restore error report.
Definition: hdf5tools.cpp:681
Here is the call graph for this function:

◆ checkDatasetReadable()

std::pair< bool, std::size_t > Gambit::Printers::HDF5::checkDatasetReadable ( hid_t  location,
const std::string &  dsetname 
)

Check if a dataset exists and can be read from fully (Reads through entire dataset to make sure! May take some time)

Definition at line 432 of file hdf5tools.cpp.

References closeDataset(), Gambit::LogTags::err, H5_OUTPUT_TYPES, hid_t, LOCAL_INFO, openDataset(), Gambit::Printers::printer_error(), and RUN_TYPE_DEPENDENT_CHECK.

Referenced by Gambit::Printers::HDF5Printer2::check_consistency(), and checkGroupReadable().

433  {
434  std::pair<bool,std::size_t> readable_info(false,0);
435  hid_t dataset_id = openDataset(location, dsetname);
436  if(dataset_id<0)
437  {
438  //msg += "Failed to open dataset";
439  }
440  else
441  {
442  hid_t datatype_id = H5Dget_type(dataset_id);
443  if(datatype_id<0)
444  {
445  //msg += "Failed to obtain type of dataset";
446  }
447  else
448  {
449  // Need buffers of various types depending of type of dataset.
450  // Can achieve this with some macros and a templated helper function
451  #define RUN_TYPE_DEPENDENT_CHECK(r,data,elem) \
452  if( H5Tequal(datatype_id, get_hdf5_data_type<elem>::type()) )\
453  {\
454  readable_info = _checkDatasetReadable_helper<elem>(dataset_id,dsetname);\
455  }\
456  else
457  BOOST_PP_SEQ_FOR_EACH(RUN_TYPE_DEPENDENT_CHECK, _, H5_OUTPUT_TYPES)
458  #undef RUN_TYPE_DEPENDENT_CHECK
459  {
460  std::ostringstream err;
461  err << "Did not recognise retrieved HDF5 type for dataset '"<<dsetname<<"'! This may indicate a bug in the GAMBIT HDF5 tools library, please report it.";
462  printer_error().raise(LOCAL_INFO,err.str());
463  }
464  }
465  }
466  closeDataset(dataset_id);
467  return readable_info;
468  }
hid_t openDataset(hid_t dset_id, const std::string &name, bool error_off=false)
Dataset and dataspace manipulation.
Definition: hdf5tools.cpp:691
#define RUN_TYPE_DEPENDENT_CHECK(r, data, elem)
EXPORT_SYMBOLS error & printer_error()
Printer errors.
#define LOCAL_INFO
Definition: local_info.hpp:34
hid_t closeDataset(hid_t dset_id)
Close dataset.
#define H5_OUTPUT_TYPES
Definition: hdf5tools.hpp:244
Here is the call graph for this function:
Here is the caller graph for this function:

◆ checkFileReadable() [1/2]

bool Gambit::Printers::HDF5::checkFileReadable ( const std::string &  fname,
std::string &  msg 
)

Check if hdf5 file exists and can be opened in read/write mode.

Check if hdf5 file exists and can be opened in read mode.

Definition at line 268 of file hdf5tools.cpp.

References errorsOff(), errorsOn(), H5P_GAMBIT(), hid_t, LOCAL_INFO, and Gambit::Printers::printer_error().

Referenced by checkFileReadable(), Gambit::Printers::HDF5Printer::common_constructor(), Gambit::Printers::HDF5::hdf5_stuff::Enter_Aux_Parameters(), Gambit::Printers::HDF5::hdf5_stuff::hdf5_stuff(), Gambit::Printers::HDF5Printer2::HDF5Printer2(), and Gambit::Printers::HDF5Printer::prepare_and_combine_tmp_files().

269  {
270  bool readable(false);
271 
272  errorsOff();
273  hid_t file_id = H5Fopen(fname.c_str(), H5F_ACC_RDONLY, H5P_GAMBIT);
274  errorsOn();
275  if(file_id < 0)
276  {
277  readable=false;
278  std::ostringstream errmsg;
279  errmsg<<"H5Fopen failed (tried to open '"<<fname<<"')";
280  msg = errmsg.str();
281  }
282  else
283  {
284  /* everything fine, close the file */
285  herr_t status = H5Fclose(file_id);
286  if(status<0)
287  {
288  std::ostringstream errmsg;
289  errmsg << "Failed to properly close HDF5 file after successfully checking that it was readable! ("<<fname<<")";
290  printer_error().raise(LOCAL_INFO, errmsg.str());
291  }
292  readable=true;
293  }
294  // DEBUG
295  #ifdef HDF5_DEBUG
296  std::cout<<"Checked that file "<<fname<<" was readable (had RDONLY access and ID "<<file_id<<")"<<std::endl;
297  #endif
298  return readable;
299  }
EXPORT_SYMBOLS error & printer_error()
Printer errors.
#define LOCAL_INFO
Definition: local_info.hpp:34
const hid_t H5P_GAMBIT(create_GAMBIT_fapl())
Const global for the GAMBIT fapl.
void errorsOff()
Silence error report (e.g. while probing for file existence)
Definition: hdf5tools.cpp:671
void errorsOn()
Restore error report.
Definition: hdf5tools.cpp:681
Here is the call graph for this function:
Here is the caller graph for this function:

◆ checkFileReadable() [2/2]

bool Gambit::Printers::HDF5::checkFileReadable ( const std::string &  fname)
inline

Thin wrapper for the above to discard failure message.

Definition at line 96 of file hdf5tools.hpp.

References checkFileReadable(), checkGroupReadable(), and hid_t.

97  { std::string garbage; return checkFileReadable(fname, garbage); }
bool checkFileReadable(const std::string &fname)
Thin wrapper for the above to discard failure message.
Definition: hdf5tools.hpp:96
Here is the call graph for this function:

◆ checkGroupReadable() [1/2]

bool Gambit::Printers::HDF5::checkGroupReadable ( hid_t  location,
const std::string &  groupname,
std::string &  msg 
)

Check if a group exists and can be accessed.

Definition at line 302 of file hdf5tools.cpp.

References errorsOff(), errorsOn(), H5Gclose, hid_t, LOCAL_INFO, and Gambit::Printers::printer_error().

Referenced by checkFileReadable(), checkGroupReadable(), Gambit::Printers::HDF5Printer::common_constructor(), and Gambit::Printers::HDF5Printer2::HDF5Printer2().

303  {
304  hid_t group_id;
305  bool readable(false);
306 
307  errorsOff();
308  group_id = H5Gopen2(location, groupname.c_str(), H5P_DEFAULT);
309  errorsOn();
310  if(group_id < 0)
311  {
312  readable=false;
313  std::ostringstream errmsg;
314  errmsg<<"H5Gopen failed (tried to open '"<<groupname<<"' from location with id "<<location<<")";
315  msg = errmsg.str();
316  }
317  else
318  {
319  /* everything fine, close the group */
320  herr_t status = H5Gclose(group_id);
321  if(status<0)
322  {
323  std::ostringstream errmsg;
324  errmsg << "Failed to properly close HDF5 group after successfully checking that it was readable! ("<<groupname<<")";
325  printer_error().raise(LOCAL_INFO, errmsg.str());
326  }
327  readable=true;
328  }
329  return readable;
330  }
EXPORT_SYMBOLS error & printer_error()
Printer errors.
#define LOCAL_INFO
Definition: local_info.hpp:34
void errorsOff()
Silence error report (e.g. while probing for file existence)
Definition: hdf5tools.cpp:671
void errorsOn()
Restore error report.
Definition: hdf5tools.cpp:681
Here is the call graph for this function:
Here is the caller graph for this function:

◆ checkGroupReadable() [2/2]

bool Gambit::Printers::HDF5::checkGroupReadable ( hid_t  location,
const std::string &  groupname 
)
inline

Thin wrapper for the above to discard failure message.

Definition at line 102 of file hdf5tools.hpp.

References checkDatasetReadable(), checkGroupReadable(), closeDataset(), closeGroup, closeSpace(), closeType(), createFile(), createGroup(), errorsOff(), errorsOn(), getH5DatasetType(), getName(), getSimpleExtentNpoints, getSpace, group, hid_t, isDataSet(), lsGroup(), openDataset(), openGroup(), and selectChunk().

103  { std::string garbage; return checkGroupReadable(location, groupname, garbage); }
bool checkGroupReadable(hid_t location, const std::string &groupname)
Thin wrapper for the above to discard failure message.
Definition: hdf5tools.hpp:102
Here is the call graph for this function:

◆ closeDataset()

hid_t Gambit::Printers::HDF5::closeDataset ( hid_t  dset_id)

◆ closeFile()

hid_t Gambit::Printers::HDF5::closeFile ( hid_t  file)

Close hdf5 file.

Definition at line 92 of file hdf5tools.cpp.

References Gambit::EOM, hid_t, LOCAL_INFO, Gambit::logger(), Gambit::Printers::printer_error(), Gambit::LogTags::repeat_to_cerr, and Gambit::LogTags::warn.

Referenced by Gambit::Printers::HDF5Printer2::check_consistency(), Gambit::Printers::HDF5MasterBuffer::close_and_unlock_file(), Gambit::Printers::HDF5Printer::common_constructor(), Gambit::Printers::HDF5::hdf5_stuff::Enter_Aux_Parameters(), Gambit::Printers::HDF5Printer::finalise(), Gambit::Printers::HDF5::hdf5_stuff::hdf5_stuff(), Gambit::Printers::HDF5Printer2::HDF5Printer2(), and Gambit::Printers::HDF5File::~HDF5File().

93  {
94  if(id < 0)
95  {
96  std::ostringstream errmsg;
97  errmsg << "Failed to close HDF5 file with ID "<<id<<"! The supplied id does not point to a successfully opened file.";
98  printer_error().raise(LOCAL_INFO, errmsg.str());
99  }
100 
101  // Check for any open objects! These should all be closed before the file is closed for maximum safety
102  ssize_t count = H5Fget_obj_count(id, H5F_OBJ_ALL);
103  if(count > 1)
104  {
106  logger() << "Warning! "<<count<<" open objects detected when closing HDF5 file with ID "<<id<<"! Please check your code and ensure that all datasets, groups, selections, etc. are closed before closing the files they belong to."<<std::endl;
107  logger() << "Beginning analysis of open objects..."<<std::endl;
108  count = H5Fget_obj_count(id, H5F_OBJ_FILE);
109  if(count>1) logger() << " "<<count<<" H5F_OBJ_FILE detected (should only be 1, for the open file itself)"<<std::endl;
110  count = H5Fget_obj_count(id, H5F_OBJ_GROUP);
111  if(count>0) logger() << " "<<count<<" H5F_OBJ_GROUP detected"<<std::endl;
112  count = H5Fget_obj_count(id, H5F_OBJ_DATASET);
113  if(count>0) logger() << " "<<count<<" H5F_OBJ_DATASET detected"<<std::endl;
114  count = H5Fget_obj_count(id, H5F_OBJ_DATATYPE);
115  if(count>0) logger() << " "<<count<<" H5F_OBJ_DATATYPE detected"<<std::endl;
116  count = H5Fget_obj_count(id, H5F_OBJ_ATTR);
117  if(count>0) logger() << " "<<count<<" H5F_OBJ_ATTR detected"<<std::endl;
118  logger()<<EOM;
119  }
120 
121  hid_t out_id = H5Fclose(id);
122  if(out_id < 0)
123  {
124  std::ostringstream errmsg;
125  errmsg << "Failed to close HDF5 file with ID "<<id<<"! See HDF5 error output for more details.";
126  printer_error().raise(LOCAL_INFO, errmsg.str());
127  }
128  #ifdef HDF5_DEBUG
129  std::cout<<"Called H5Fclose on file with ID "<<id<<std::endl;
130  #endif
131  return out_id;
132  }
EXPORT_SYMBOLS error & printer_error()
Printer errors.
#define LOCAL_INFO
Definition: local_info.hpp:34
const Logging::endofmessage EOM
Explicit const instance of the end of message struct in Gambit namespace.
Definition: logger.hpp:99
Logging::LogMaster & logger()
Function to retrieve a reference to the Gambit global log object.
Definition: logger.cpp:95
Here is the call graph for this function:
Here is the caller graph for this function:

◆ closeGroup()

hid_t Gambit::Printers::HDF5::closeGroup ( hid_t  group)

Close group.

◆ closeSpace()

◆ closeType()

hid_t Gambit::Printers::HDF5::closeType ( hid_t  type_id)

Release datatype identifier.

Referenced by checkGroupReadable(), Gambit::Printers::HDF5Reader::get_type(), and getH5DatasetType().

Here is the caller graph for this function:

◆ combine_hdf5_files()

void Gambit::Printers::HDF5::combine_hdf5_files ( const std::string  output_file,
const std::string &  base_file_name,
const std::string &  group,
const size_t  num,
const bool  resume,
const bool  cleanup,
const bool  skip,
const std::vector< std::string >  input_files = std::vector<std::string>() 
)
inline

Definition at line 455 of file hdf5_combine_tools.hpp.

References Gambit::Printers::HDF5::hdf5_stuff::Enter_Aux_Parameters(), find_temporary_files(), get_RA_write_hash(), and hid_t.

Referenced by Gambit::Printers::HDF5Printer::combine_output(), and main().

456  {
457  hdf5_stuff stuff(base_file_name, output_file, group, num, cleanup, skip, input_files);
458 
459  stuff.Enter_Aux_Parameters(output_file, resume);
460  }
Here is the call graph for this function:
Here is the caller graph for this function:

◆ create_GAMBIT_fapl()

hid_t Gambit::Printers::HDF5::create_GAMBIT_fapl ( )

GAMBIT default file access property list.

Definition at line 43 of file hdf5tools.cpp.

References H5P_GAMBIT(), and hid_t.

44  {
45  hid_t fapl(H5Pcreate(H5P_FILE_ACCESS)); // Copy defaults
46  #ifdef HDF5_DEBUG
47  std::cout<<"HDF5 version:"<<H5_VERS_MAJOR<<"."<<H5_VERS_MINOR<<"."<<H5_VERS_RELEASE<<std::endl;
48  #endif
49  #if (H5_VERS_MAJOR > 1) || \
50  (H5_VERS_MAJOR == 1) && H5_VERS_MINOR > 10 || \
51  (H5_VERS_MAJOR == 1) && H5_VERS_MINOR == 10 && H5_VERS_RELEASE >= 1
52  hbool_t value = 1; // true?
53  // This function does not appear before v 1.10.1, however it is
54  // pretty crucial, at least in my version of HDF5, for keeping the
55  // metadata cache from consuming all my RAM. However, Anders commented
56  // it out with his older HDF5 version and still had no RAM problem.
57  // So it might be ok to just remove it for older versions.
58  // However, if you see RAM blowouts and your HDF5 version is old,
59  // then this is probably the reason.
60  H5Pset_evict_on_close(fapl, value); // Set evict_on_close = true
61  #ifdef HDF5_DEBUG
62  std::cout <<"GAMBIT fapl used!"<<std::endl; // Check that this code is built...
63  #endif
64  #endif
65 
66  return fapl;
67  }
Here is the call graph for this function:

◆ createFile()

hid_t Gambit::Printers::HDF5::createFile ( const std::string &  fname)

Create hdf5 file (always overwrite existing files)

Definition at line 471 of file hdf5tools.cpp.

References H5P_GAMBIT(), hid_t, LOCAL_INFO, and Gambit::Printers::printer_error().

Referenced by checkGroupReadable().

472  {
473  hid_t file_id = H5Fcreate(fname.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, H5P_GAMBIT);
474  if(file_id < 0)
475  {
476  /* Still no good; error */
477  std::ostringstream errmsg;
478  errmsg << "Failed to create HDF5 file '"<<fname<<"'!";
479  printer_error().raise(LOCAL_INFO, errmsg.str());
480  }
481  return file_id;
482  }
EXPORT_SYMBOLS error & printer_error()
Printer errors.
#define LOCAL_INFO
Definition: local_info.hpp:34
const hid_t H5P_GAMBIT(create_GAMBIT_fapl())
Const global for the GAMBIT fapl.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ createGroup()

hid_t Gambit::Printers::HDF5::createGroup ( hid_t  location,
const std::string &  name 
)

Create a group inside the specified location.

Definition at line 486 of file hdf5tools.cpp.

References hid_t, LOCAL_INFO, and Gambit::Printers::printer_error().

Referenced by checkGroupReadable().

487  {
488  hid_t group_id;
489 
490  group_id = H5Gcreate2(location, name.c_str(), H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
491  if(group_id<0)
492  {
493  std::ostringstream errmsg;
494  errmsg << "Error creating HDF5 group '"<<name<<"'";
495  printer_error().raise(LOCAL_INFO, errmsg.str());
496  }
497  return group_id;
498  }
EXPORT_SYMBOLS error & printer_error()
Printer errors.
#define LOCAL_INFO
Definition: local_info.hpp:34
Here is the call graph for this function:
Here is the caller graph for this function:

◆ Enter_HDF5()

template<class U , typename... T>
void Gambit::Printers::HDF5::Enter_HDF5 ( hid_t  dataset,
T &...  params 
)
inline

Definition at line 341 of file hdf5_combine_tools.hpp.

References combine_hdf5::dtype, hid_t, LOCAL_INFO, Gambit::Printers::printer_error(), and run().

Referenced by type_ret().

342  {
343  if(dataset<0)
344  {
345  std::ostringstream errmsg;
346  errmsg << "Invalid dataset supplied to Enter_HDF5 routine!";
347  printer_error().raise(LOCAL_INFO, errmsg.str());
348  }
349 
350  hid_t dtype = H5Dget_type(dataset);
351  if(dtype<0)
352  {
353  std::ostringstream errmsg;
354  errmsg << "Failed to detect type for dataset provides as argument for Enter_HDF5 routine!";
355  printer_error().raise(LOCAL_INFO, errmsg.str());
356  }
357 
358  //H5T_class_t cl = H5Tget_class(dtype);
359  hid_t type = H5Tget_native_type(dtype, H5T_DIR_DESCEND);
360  if(type<0)
361  {
362  std::ostringstream errmsg;
363  errmsg << "Failed to detect native type for dataset provides as argument for Enter_HDF5 routine!";
364  printer_error().raise(LOCAL_INFO, errmsg.str());
365  }
366 
367  if (H5Tequal(type, get_hdf5_data_type<float>::type()))
368  {
369  U::run(float(), dataset, params...);
370  }
371  else if (H5Tequal(type, get_hdf5_data_type<double>::type()))
372  {
373  U::run(double(), dataset, params...);
374  }
375  else if (H5Tequal(type, get_hdf5_data_type<long double>::type()))
376  {
377  U::run(type_ret<long double>(), dataset, params...);
378  }
379  else if (H5Tequal(type, get_hdf5_data_type<char>::type()))
380  {
381  U::run(char(), dataset, params...);
382  }
383  else if (H5Tequal(type, get_hdf5_data_type<short>::type()))
384  {
385  U::run(short(), dataset, params...);
386  }
387  else if (H5Tequal(type, get_hdf5_data_type<int>::type()))
388  {
389  U::run(int(), dataset, params...);
390  }
391  else if (H5Tequal(type, get_hdf5_data_type<long>::type()))
392  {
393  U::run(long(), dataset, params...);
394  }
395  else if (H5Tequal(type, get_hdf5_data_type<long long>::type()))
396  {
397  U::run(type_ret<long long>(), dataset, params...);
398  }
399  else if (H5Tequal(type, get_hdf5_data_type<unsigned char>::type()))
400  {
401  U::run(type_ret<unsigned char>(), dataset, params...);
402  }
403  else if (H5Tequal(type, get_hdf5_data_type<unsigned short>::type()))
404  {
405  U::run(type_ret<unsigned short>(), dataset, params...);
406  }
407  else if (H5Tequal(type, get_hdf5_data_type<unsigned int>::type()))
408  {
409  U::run(type_ret<unsigned int>(), dataset, params...);
410  }
411  else if (H5Tequal(type, get_hdf5_data_type<unsigned long>::type()))
412  {
413  U::run(type_ret<unsigned long>(), dataset, params...);
414  }
415  else if (H5Tequal(type, get_hdf5_data_type<unsigned long long>::type()))
416  {
417  U::run(type_ret<unsigned long long>(), dataset, params...);
418  }
419  else
420  {
421  std::ostringstream errmsg;
422  errmsg << "Could not deduce input hdf5 parameter type.";
423  printer_error().raise(LOCAL_INFO, errmsg.str());
424  }
425 
426  H5Tclose(dtype);
427  }
EXPORT_SYMBOLS error & printer_error()
Printer errors.
#define LOCAL_INFO
Definition: local_info.hpp:34
void run(bool, bool, bool, int, double, double, int, int, int, int, int, double, char[], int, int[], bool, bool, bool, bool, double, int, double(*)(double *, int, int, void *), void(*)(int, int, int, double *, double *, double *, double, double, double, void *), void *)
Here is the call graph for this function:
Here is the caller graph for this function:

◆ errorsOff()

void Gambit::Printers::HDF5::errorsOff ( )

Silence error report (e.g. while probing for file existence)

Silence error report (e.g.

while probing for file existence) Just silences default error stack, since we aren't using anything else TESTING! I changed from using H5Eget_auto to H5Eget_auto2 If that still causes errors, try switching to H5Eget_auto1 and let me know if it works :)

Definition at line 671 of file hdf5tools.cpp.

Referenced by _checkDatasetReadable_helper(), checkFileReadable(), checkGroupReadable(), Gambit::Printers::HDF5::hdf5_stuff::Enter_Aux_Parameters(), Gambit::Printers::HDF5::hdf5_stuff::hdf5_stuff(), openFile(), and openGroup().

672  {
673  /* Save old error handler */
674  H5Eget_auto2(H5E_DEFAULT, &old_func, &old_client_data);
675 
676  /* Turn off error handling */
677  H5Eset_auto2(H5E_DEFAULT, NULL, NULL);
678  }
group H5E_auto2_t old_func
global error variables (handler)
Definition: hdf5tools.cpp:655
Here is the caller graph for this function:

◆ errorsOn()

void Gambit::Printers::HDF5::errorsOn ( )

Restore error report.

Definition at line 681 of file hdf5tools.cpp.

References old_client_data, and old_func.

Referenced by _checkDatasetReadable_helper(), checkFileReadable(), checkGroupReadable(), Gambit::Printers::HDF5::hdf5_stuff::Enter_Aux_Parameters(), Gambit::Printers::HDF5::hdf5_stuff::hdf5_stuff(), openFile(), and openGroup().

682  {
683  /* Restore previous error handler */
684  H5Eset_auto2(H5E_DEFAULT, old_func, old_client_data);
685  }
group H5E_auto2_t old_func
global error variables (handler)
Definition: hdf5tools.cpp:655
Here is the caller graph for this function:

◆ find_temporary_files() [1/2]

std::pair< std::vector< std::string >, std::vector< size_t > > Gambit::Printers::HDF5::find_temporary_files ( const std::string &  finalfile)

Search for temporary files to be combined.

Definition at line 1218 of file hdf5_combine_tools.cpp.

Referenced by Gambit::Printers::HDF5Printer::clear_previous_points(), combine_hdf5_files(), Gambit::Printers::HDF5Printer::find_temporary_files(), and main().

1219  {
1220  size_t max_i = 0;
1221  return find_temporary_files(finalfile, max_i);
1222  }
std::pair< std::vector< std::string >, std::vector< size_t > > find_temporary_files(const std::string &finalfile)
Search for temporary files to be combined.
Here is the caller graph for this function:

◆ find_temporary_files() [2/2]

std::pair< std::vector< std::string >, std::vector< size_t > > Gambit::Printers::HDF5::find_temporary_files ( const std::string &  finalfile,
size_t &  max_i 
)

Search for temporary files to be combined.

Definition at line 1225 of file hdf5_combine_tools.cpp.

References Gambit::Utils::base_name(), Gambit::Utils::dir_name(), Gambit::Printers::HDF5::hdf5_stuff::files, Gambit::Utils::isInteger(), Gambit::Utils::ls_dir(), and combine_hdf5::rank.

1226  {
1227  // Autodetect temporary files from previous run.
1228  std::string output_dir = Utils::dir_name(finalfile);
1229  std::vector<std::string> files = Utils::ls_dir(output_dir);
1230  std::string tmp_base(Utils::base_name(finalfile) + "_temp_");
1231  std::vector<size_t> ranks;
1232  std::vector<std::string> result;
1233 
1234  //std::cout << "Matching against: " <<tmp_base<<std::endl;
1235  for(auto it=files.begin(); it!=files.end(); ++it)
1236  {
1237  //std::cout << (*it) << std::endl;
1238  //std::cout << it->substr(0,tmp_base.length()) << std::endl;
1239  if (it->compare(0, tmp_base.length(), tmp_base) == 0)
1240  {
1241  // Matches format of temporary file! Extract the rank that produced it
1242  std::stringstream ss;
1243  ss << it->substr(tmp_base.length());
1244  if(Utils::isInteger(ss.str())) // Only do this for files where the remainder of the string is just an integer (i.e. not the combined files etc.)
1245  {
1246  size_t rank;
1247  ss >> rank;
1248  if(rank>max_i) max_i = rank; // Look for highest rank output
1249  //std::cout << "Match! "<< ss.str() << " : " << rank << std::endl;
1250  // TODO: check for failed read
1251  ranks.push_back(rank);
1252  result.push_back(output_dir+"/"+*it);
1253  }
1254  }
1255  }
1256 
1257  // Check if all temporary files found (i.e. if output from some rank is missing)
1258  std::vector<size_t> missing;
1259  for(size_t i=0; i<max_i; ++i)
1260  {
1261  if(std::find(ranks.begin(), ranks.end(), i) == ranks.end())
1262  { missing.push_back(i); }
1263  }
1264 
1265  // Return the list of missing tmp files, the caller can decide to throw an error if they like
1266  return std::make_pair(result,missing);
1267  }
EXPORT_SYMBOLS str dir_name(const str &path)
Get directory name from full path+filename (POSIX)
EXPORT_SYMBOLS bool isInteger(const std::string &)
Check if a string represents an integer From: http://stackoverflow.com/a/2845275/1447953.
EXPORT_SYMBOLS std::vector< str > ls_dir(const str &dir)
Return a vector of strings listing the contents of a directory (POSIX)
EXPORT_SYMBOLS str base_name(const str &path)
Get file name from full path+filename (POSIX)
Here is the call graph for this function:

◆ get_dset_names()

std::vector<std::string> Gambit::Printers::HDF5::get_dset_names ( hid_t  group_id)

Definition at line 73 of file hdf5_combine_tools.cpp.

Referenced by Gambit::Printers::HDF5::hdf5_stuff::Enter_Aux_Parameters(), and Gambit::Printers::HDF5::hdf5_stuff::hdf5_stuff().

74  {
75  std::vector<std::string> outnames;
76  H5G_info_t group_info;
77  H5Gget_info(group_id, &group_info );
78  hsize_t nlinks = group_info.nlinks; // Number of objects in the group
79  for(hsize_t i=0; i<nlinks; i++)
80  {
81  int otype = H5Gget_objtype_by_idx(group_id, i);
82  switch (otype)
83  {
84  case H5G_DATASET:
85  {
86  char name_buffer[1000];
87  H5Gget_objname_by_idx(group_id, i, name_buffer, (size_t)1000);
88  std::string strname(name_buffer);
89  // std::cout << "Found "<<strname<<std::endl;
90  if (strname.find("_isvalid") == std::string::npos && strname != "RA_pointID" && strname != "RA_MPIrank")
91  outnames.push_back(strname);
92  break;
93  }
94  default:
95  break;
96  }
97  }
98  return outnames;
99  }
Here is the caller graph for this function:

◆ get_RA_write_hash()

std::unordered_map< PPIDpair, unsigned long long, PPIDHash, PPIDEqual > Gambit::Printers::HDF5::get_RA_write_hash ( hid_t  group_id,
std::unordered_set< PPIDpair, PPIDHash, PPIDEqual > &  left_to_match 
)

Helper function to create output hash map for RA points note: left_to_match points will be erased as we go, and are passed by reference, so will be erased in calling context also.

Definition at line 1078 of file hdf5_combine_tools.cpp.

References Gambit::Printers::DataSetInterfaceBase< T, RECORDRANK, CHUNKLENGTH >::dset_length(), combine_hdf5::dset_length, Gambit::Printers::DataSetInterfaceScalar< T, CHUNKLENGTH >::get_chunk(), LOCAL_INFO, and Gambit::Printers::printer_error().

Referenced by combine_hdf5_files(), and Gambit::Printers::HDF5::hdf5_stuff::Enter_Aux_Parameters().

1079  {
1080  std::unordered_map<PPIDpair, unsigned long long, PPIDHash,PPIDEqual> output_hash;
1081 
1082  // Chunking variables
1083  static const std::size_t CHUNKLENGTH = 10000; // Should be a reasonable value
1084 
1085  // Interfaces for the datasets
1086  // Make sure the types used here don't get out of sync with the types used to write the original datasets
1087  // We open the datasets in "resume" mode to access existing dataset, and make "const" to disable writing of new data. i.e. "Read-only" mode.
1088  // TODO: this can probably be streamlined once I write the HDF5 reader, can consolidate some reading routines.
1089  const DataSetInterfaceScalar<unsigned long, CHUNKLENGTH> pointIDs(group_id, "pointID", true,'r');
1090  const DataSetInterfaceScalar<int, CHUNKLENGTH> pointIDs_isvalid (group_id, "pointID_isvalid", true,'r');
1091  const DataSetInterfaceScalar<int, CHUNKLENGTH> mpiranks (group_id, "MPIrank", true,'r');
1092  const DataSetInterfaceScalar<int, CHUNKLENGTH> mpiranks_isvalid (group_id, "MPIrank_isvalid", true,'r');
1093 
1094  // Error check lengths. This should already have been done for all datasets in the group, but
1095  // we will double-check these four here.
1096  const std::size_t dset_length = pointIDs.dset_length();
1097  const std::size_t dset_length2 = pointIDs_isvalid.dset_length();
1098  const std::size_t dset_length3 = mpiranks.dset_length();
1099  const std::size_t dset_length4 = mpiranks_isvalid.dset_length();
1100  if( (dset_length != dset_length2)
1101  or (dset_length3 != dset_length4)
1102  or (dset_length != dset_length3) )
1103  {
1104  std::ostringstream errmsg;
1105  errmsg << "Error scanning combined output for RA point locations! Unequal dataset lengths detected in pointID and MPIrank datasets:" <<std::endl;
1106  errmsg << " pointIDs.dset_length() = " << dset_length << std::endl;
1107  errmsg << " pointIDs_isvalid.dset_length() = " << dset_length2 << std::endl;
1108  errmsg << " mpiranks.dset_length() = " << dset_length3 << std::endl;
1109  errmsg << " mpiranks_isvalid.dset_length() = " << dset_length4 << std::endl;
1110  errmsg << "This indicates either a bug in the HDF5 combine code, please report it.";
1111  printer_error().raise(LOCAL_INFO, errmsg.str());
1112  }
1113 
1114  //std::cout<<"New combined output dataset length: "<<dset_length<<std::endl;
1115 
1116  // Compute number of chunks
1117  const std::size_t NCHUNKS = dset_length / CHUNKLENGTH; // Number of FULL chunks
1118  const std::size_t REMAINDER = dset_length - (NCHUNKS*CHUNKLENGTH); // leftover after last full chunk
1119 
1120  std::size_t NCHUNKIT; // Number of chunk iterations to perform
1121  if(REMAINDER==0) { NCHUNKIT = NCHUNKS; }
1122  else { NCHUNKIT = NCHUNKS+1; } // Need an extra iteration to deal with incomplete chunk
1123 
1124  //std::cout<<"Will parse this output in "<<NCHUNKIT<<" chunks of size "<<CHUNKLENGTH<<std::endl;
1125 
1126  // Iterate through dataset in chunks
1127  for(std::size_t i=0; i<NCHUNKIT; ++i)
1128  {
1129  std::size_t offset = i*CHUNKLENGTH;
1130  std::size_t length;
1131  if(i==NCHUNKS){ length = REMAINDER; }
1132  else { length = CHUNKLENGTH; }
1133 
1134  //std::cout<<" Offset for chunk "<<i<<" is "<<offset<<". Length is "<<length<<std::endl;
1135 
1136  const std::vector<unsigned long> pID_chunk = pointIDs.get_chunk(offset,length);
1137  const std::vector<int> pIDvalid_chunk = pointIDs_isvalid.get_chunk(offset,length);
1138  const std::vector<int> rank_chunk = mpiranks.get_chunk(offset,length);
1139  const std::vector<int> rankvalid_chunk = mpiranks_isvalid.get_chunk(offset,length);
1140 
1141  // Check that retrieved lengths make sense
1142  if (pID_chunk.size() != CHUNKLENGTH)
1143  {
1144  if(not (i==NCHUNKS and pID_chunk.size()==REMAINDER) )
1145  {
1146  std::ostringstream errmsg;
1147  errmsg << "Error scanning combined pointID and MPIrank datasets! Size of chunk vector retrieved from pointID dataset ("<<pID_chunk.size()<<") does not match CHUNKLENGTH ("<<CHUNKLENGTH<<"), nor the expected remainder for the last chunk ("<<REMAINDER<<"). This probably indicates a bug in the DataSetInterfaceScalar.get_chunk routine, please report it. Error occurred while reading chunk i="<<i<<std::endl;
1148  printer_error().raise(LOCAL_INFO, errmsg.str());
1149  }
1150  }
1151  if( (pID_chunk.size() != pIDvalid_chunk.size())
1152  or (rank_chunk.size() != rankvalid_chunk.size())
1153  or (pID_chunk.size() != rank_chunk.size()) )
1154  {
1155  std::ostringstream errmsg;
1156  errmsg << "Error preparing to combine RA points into output dataset! Unequal chunk lengths retrieved while iterating through in pointID and MPIrank datasets:" <<std::endl;
1157  errmsg << " pID_chunk.size() = " << pID_chunk.size() << std::endl;
1158  errmsg << " pIDvalid_chunk.size() = " << pIDvalid_chunk.size() << std::endl;
1159  errmsg << " rank_chunk.size() = " << rank_chunk.size() << std::endl;
1160  errmsg << " rankvalid_chunk.size()= " << rankvalid_chunk.size() << std::endl;
1161  errmsg << " CHUNKLENGTH = " << CHUNKLENGTH << std::endl;
1162  errmsg << "This indicates a bug in the HDF5 combine code, please report it. Error occurred while reading chunk i="<<i<<std::endl;
1163  printer_error().raise(LOCAL_INFO, errmsg.str());
1164  }
1165 
1166  // Iterate within the chunk
1167  for(std::size_t j=0; j<length; ++j)
1168  {
1169  //Check validity flags agree
1170  if(pIDvalid_chunk[j] != rankvalid_chunk[j])
1171  {
1172  std::ostringstream errmsg;
1173  errmsg << "Error! Incompatible validity flags detected in pointID_isvalid and MPIrank_isvalid datasets at position j="<<j<<" in chunk i="<<i<<"(with CHUNKLENGTH="<<CHUNKLENGTH<<"). Specifically:"<<std::endl;
1174  errmsg << " pIDvalid_chunk[j] = " << pIDvalid_chunk[j] << std::endl;
1175  errmsg << " rankvalid_chunk[j] = " << rankvalid_chunk[j] << std::endl;
1176  errmsg << "This most likely indicates a bug in the HDF5 combine code, please report it.";
1177  printer_error().raise(LOCAL_INFO, errmsg.str());
1178  }
1179  // @{ Debugging: see what points exist in the output dataset (for rank 127)
1180  //if(rank_chunk[j]==127)
1181  //{
1182  // std::cout << " Scanned point "<<pID_chunk[j]<<" (rank "<<rank_chunk[j]<<", valid="<<pIDvalid_chunk[j]<<") in combined output file"<<std::endl;
1183  //}
1184  // @}
1185 
1186  // Check for hash match if entry is marked as "valid"
1187  if(rankvalid_chunk[j])
1188  {
1189  // Check if this point is in our list of points to be matched
1190  PPIDpair this_point(pID_chunk[j],rank_chunk[j]);
1191  std::unordered_set<PPIDpair,PPIDHash,PPIDEqual>::iterator match = left_to_match.find(this_point);
1192  if(match != left_to_match.end())
1193  {
1194  // Found a match! Add its index to the hash.
1195  output_hash[*match] = offset + j;
1196  // Delete it from the list of points that need to be matched (note, multiple entries in output file not allowed)
1197  left_to_match.erase(match);
1198  }
1199  }
1200  // else continue iteration
1201  }
1202  }
1203 
1204  // Check that all the matches were found!
1205  if( left_to_match.size() > 0 )
1206  {
1207  std::ostringstream errmsg;
1208  errmsg << "Error generating hash map for Auxilliary parameter copy! Failed to find matches in primary datasets for all auxilliary data! There were "<<left_to_match.size()<<" unmatched auxilliary points. Unmatched points follow:" << std::endl;
1209  for(auto it=left_to_match.begin(); it!=left_to_match.end(); ++it)
1210  {
1211  errmsg << " rank: "<<it->rank<<", pointID: "<<it->pointID<< std::endl;
1212  }
1213  printer_error().raise(LOCAL_INFO, errmsg.str());
1214  }
1215  return output_hash;
1216  }
EXPORT_SYMBOLS error & printer_error()
Printer errors.
#define LOCAL_INFO
Definition: local_info.hpp:34
Here is the call graph for this function:
Here is the caller graph for this function:

◆ getChunk() [1/2]

template<class T >
std::vector<T> Gambit::Printers::HDF5::getChunk ( const hid_t  dset_id,
std::size_t  offset,
std::size_t  length 
)

Retrieve a chunk of data from a simple dataset NOTE! Doesn't work for T=bool! Have a custom specialisation in the source file for that.

Definition at line 179 of file hdf5tools.hpp.

References hid_t, inttype_from_h5type(), is_float_type(), LOCAL_INFO, Gambit::Printers::printer_error(), and selectChunk().

180  {
181  // Buffer to receive data (and return from function)
182  std::vector<T> chunkdata(length);
183 
184  // Select hyperslab
185  std::pair<hid_t,hid_t> selection_ids = selectChunk(dset_id,offset,length);
186  hid_t memspace_id = selection_ids.first;
187  hid_t dspace_id = selection_ids.second;
188 
189  // Buffer to receive data
190  void* buffer = chunkdata.data(); // pointer to contiguous memory within the buffer vector
191 
192  // Get the data from the hyperslab.
193  hid_t hdftype_id = get_hdf5_data_type<T>::type(); // It is assumed that you already know this is the right type for the dataset!
194  herr_t err_read = H5Dread(dset_id, hdftype_id, memspace_id, dspace_id, H5P_DEFAULT, buffer);
195 
196  if(err_read<0)
197  {
198  std::ostringstream errmsg;
199  errmsg << "Error retrieving chunk (offset="<<offset<<", length="<<length<<") from dataset in HDF5 file. H5Dread failed." << std::endl;
200  errmsg << " offset+length = "<< offset+length << std::endl;
201  printer_error().raise(LOCAL_INFO, errmsg.str());
202  }
203 
204  H5Sclose(dspace_id);
205  H5Sclose(memspace_id);
206 
207  return chunkdata;
208  }
EXPORT_SYMBOLS error & printer_error()
Printer errors.
#define LOCAL_INFO
Definition: local_info.hpp:34
std::pair< hid_t, hid_t > selectChunk(const hid_t dset_id, std::size_t offset, std::size_t length)
Select a simple hyperslab in a 1D dataset.
Definition: hdf5tools.cpp:733
Here is the call graph for this function:

◆ getChunk() [2/2]

template<>
std::vector< bool > Gambit::Printers::HDF5::getChunk ( const hid_t  dset_id,
std::size_t  offset,
std::size_t  length 
)

Definition at line 136 of file hdf5tools.cpp.

References hid_t, LOCAL_INFO, Gambit::Printers::printer_error(), and selectChunk().

137  {
138  // Buffer to receive data (and return from function)
139  std::vector<uint8_t> chunkdata(length);
140 
141  // Select hyperslab
142  std::pair<hid_t,hid_t> selection_ids = selectChunk(dset_id,offset,length);
143  hid_t memspace_id = selection_ids.first;
144  hid_t dspace_id = selection_ids.second;
145 
146  // Buffer to receive data
147  void* buffer = chunkdata.data(); // pointer to contiguous memory within the buffer vector
148 
149  // Get the data from the hyperslab.
150  hid_t hdftype_id = get_hdf5_data_type<bool>::type(); // It is assumed that you already know this is the right type for the dataset!
151  herr_t err_read = H5Dread(dset_id, hdftype_id, memspace_id, dspace_id, H5P_DEFAULT, buffer);
152 
153  if(err_read<0)
154  {
155  std::ostringstream errmsg;
156  errmsg << "Error retrieving chunk (offset="<<offset<<", length="<<length<<") from dataset in HDF5 file. H5Dread failed." << std::endl;
157  errmsg << " offset+length = "<< offset+length << std::endl;
158  printer_error().raise(LOCAL_INFO, errmsg.str());
159  }
160 
161  H5Sclose(dspace_id);
162  H5Sclose(memspace_id);
163 
164  std::vector<bool> chunkdata_bool;
165  for(auto it=chunkdata.begin(); it!=chunkdata.end(); ++it)
166  {
167  chunkdata_bool.push_back(*it);
168  }
169 
170  return chunkdata_bool;
171  }
EXPORT_SYMBOLS error & printer_error()
Printer errors.
#define LOCAL_INFO
Definition: local_info.hpp:34
std::pair< hid_t, hid_t > selectChunk(const hid_t dset_id, std::size_t offset, std::size_t length)
Select a simple hyperslab in a 1D dataset.
Definition: hdf5tools.cpp:733
Here is the call graph for this function:

◆ getGroupNum()

hsize_t Gambit::Printers::HDF5::getGroupNum ( hid_t  group_id)
inline

Definition at line 42 of file hdf5_combine_tools.cpp.

43  {
44  H5G_info_t group_info;
45  H5Gget_info(group_id, &group_info);
46  return group_info.nlinks;
47  }

◆ getGroups()

std::vector<std::string> Gambit::Printers::HDF5::getGroups ( std::string  groups)
inline

Definition at line 226 of file hdf5_combine_tools.cpp.

References Gambit::Printers::HDF5::hdf5_stuff::groups.

227  {
228  std::string::size_type pos = groups.find_first_of("/");
229  while (pos != std::string::npos)
230  {
231  groups[pos] = ' ';
232  pos = groups.find_first_of("/", pos + 1);
233  }
234 
235  std::stringstream ss;
236  ss << groups;
237  std::vector<std::string> ret;
238  std::string temp;
239  while (ss >> temp) ret.push_back(temp);
240 
241  return ret;
242  }

◆ getH5DatasetType()

hid_t Gambit::Printers::HDF5::getH5DatasetType ( hid_t  group_id,
const std::string &  dset_name 
)

Get type of an object in a group.

Get type of a dataset in a group NOTE: Make sure to call closeType when the ID is no longer needed!

Definition at line 631 of file hdf5tools.cpp.

References closeDataset(), closeType(), hid_t, LOCAL_INFO, openDataset(), Gambit::Printers::printer_error(), and SIMPLE_CALL().

Referenced by checkGroupReadable(), Gambit::Printers::HDF5MasterBuffer::get_all_dset_names_on_disk(), and Gambit::Printers::HDF5Reader::get_type().

632  {
633  hid_t dataset_id = openDataset(group_id, dset_name);
634  if(dataset_id<0)
635  {
636  std::ostringstream errmsg;
637  errmsg << "Failed to open dataset '"<<dset_name<<"' while attempting to check its HDF5 data type! See stderr output for more details.";
638  printer_error().raise(LOCAL_INFO, errmsg.str());
639  }
640  hid_t type_id = H5Dget_type(dataset_id);
641  if(type_id<0)
642  {
643  std::ostringstream errmsg;
644  errmsg << "Failed to get HDF5 type of dataset '"<<dset_name<<"'. See stderr output for more details.";
645  printer_error().raise(LOCAL_INFO, errmsg.str());
646  }
647  closeDataset(dataset_id);
648  return type_id;
649  }
hid_t openDataset(hid_t dset_id, const std::string &name, bool error_off=false)
Dataset and dataspace manipulation.
Definition: hdf5tools.cpp:691
EXPORT_SYMBOLS error & printer_error()
Printer errors.
#define LOCAL_INFO
Definition: local_info.hpp:34
hid_t closeDataset(hid_t dset_id)
Close dataset.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ getName()

dataset dataspace std::string Gambit::Printers::HDF5::getName ( hid_t  dset_id)

Get name of dataset.

Get dataset name.

Definition at line 723 of file hdf5tools.cpp.

Referenced by checkGroupReadable(), and Gambit::SubSpectrum::~SubSpectrum().

724  {
725  size_t len = H5Iget_name(dset_id,NULL,0);
726  char buffer[len];
727  H5Iget_name(dset_id,buffer,len+1);
728  std::string n = buffer;
729  return n;
730  }
Here is the caller graph for this function:

◆ getSimpleExtentNpoints()

hssize_t Gambit::Printers::HDF5::getSimpleExtentNpoints ( hid_t  dset_id)

Get simple dataspace extent.

◆ getSpace()

hid_t Gambit::Printers::HDF5::getSpace ( hid_t  dset_id)

Get dataspace.

◆ getType()

hid_t Gambit::Printers::HDF5::getType ( hid_t  dataset)
inline

Definition at line 49 of file hdf5_combine_tools.cpp.

References combine_hdf5::dtype, and hid_t.

50  {
51  hid_t dtype = H5Dget_type(dataset);
52  hid_t ret = H5Tget_native_type(dtype, H5T_DIR_DESCEND);
53  H5Tclose(dtype);
54  return ret;
55  }

◆ group_ls()

herr_t Gambit::Printers::HDF5::group_ls ( hid_t  g_id,
const char *  name,
const H5L_info_t *  ,
void op_data 
)

Definition at line 578 of file hdf5tools.cpp.

Referenced by lsGroup().

579  {
580  #ifdef HDF5_DEBUG
581  //std::cout<<"group_ls: "<<name<<std::endl;
582  //std::cout<<info->type<<" "<<H5G_DATASET<<std::endl;
583  #endif
584  std::vector<std::string>* out = static_cast<std::vector<std::string>*>(op_data);
585  // Only add names that correspond to datasets
586  H5G_stat_t statbuf;
587  H5Gget_objinfo(g_id, name, false, &statbuf);
588  if(statbuf.type == H5G_DATASET) out->push_back(name);
589  return 0;
590  }
Here is the caller graph for this function:

◆ H5P_GAMBIT()

const hid_t Gambit::Printers::HDF5::H5P_GAMBIT ( create_GAMBIT_fapl()  )

Const global for the GAMBIT fapl.

Referenced by checkFileReadable(), create_GAMBIT_fapl(), createFile(), and openFile().

Here is the caller graph for this function:

◆ inttype_from_h5type()

int Gambit::Printers::HDF5::inttype_from_h5type ( hid_t  h5type)

Definition at line 783 of file hdf5tools.cpp.

References ELSEIF, H5_OUTPUT_TYPES, LOCAL_INFO, and Gambit::Printers::printer_error().

Referenced by Gambit::Printers::HDF5MasterBuffer::get_all_dset_names_on_disk(), and getChunk().

784  {
785  #define ELSEIF(r,data,elem) \
786  else if(H5Tequal(h5type,get_hdf5_data_type<elem>::type())) \
787  { \
788  out = h5v2_type<elem>(); \
789  }
790 
791  int out;
792  if(h5type==-1)
793  {
794  std::ostringstream errmsg;
795  errmsg<<"No fixed ID assigned for this type! (h5type = "<<h5type<<")!";
796  printer_error().raise(LOCAL_INFO, errmsg.str());
797  }
798  BOOST_PP_SEQ_FOR_EACH(ELSEIF, _, H5_OUTPUT_TYPES)
799  #undef ELSEIF
800  else
801  {
802  std::ostringstream errmsg;
803  errmsg<<"Unrecognised HDF5 type (h5type = "<<h5type<<")!";
804  printer_error().raise(LOCAL_INFO, errmsg.str());
805  }
806  return out;
807  }
EXPORT_SYMBOLS error & printer_error()
Printer errors.
#define LOCAL_INFO
Definition: local_info.hpp:34
#define ELSEIF(r, data, elem)
#define H5_OUTPUT_TYPES
Definition: hdf5tools.hpp:244
Here is the call graph for this function:
Here is the caller graph for this function:

◆ is_float_type()

bool Gambit::Printers::HDF5::is_float_type ( int  inttype)

Definition at line 810 of file hdf5tools.cpp.

References Gambit::Printers::h5v2_type< double >(), Gambit::Printers::h5v2_type< float >(), Gambit::Printers::h5v2_type< int >(), Gambit::Printers::h5v2_type< long >(), Gambit::Printers::h5v2_type< long long >(), Gambit::Printers::h5v2_type< unsigned int >(), Gambit::Printers::h5v2_type< unsigned long >(), and Gambit::Printers::h5v2_type< unsigned long long >().

Referenced by getChunk(), and Gambit::Printers::HDF5Printer2::resume_reader_options().

811  {
812  bool out(false);
813  switch(inttype)
814  {
815  case h5v2_type<int >():
817  case h5v2_type<long >():
818  case h5v2_type<unsigned long >():
819  case h5v2_type<long long >():
820  case h5v2_type<unsigned long long>():
821  out = false;
822  break;
823  case h5v2_type<float >():
824  case h5v2_type<double >():
825  out = true;
826  break;
827  }
828  return out;
829  }
constexpr int h5v2_type< int >()
Definition: hdf5tools.hpp:279
constexpr int h5v2_type< unsigned int >()
Definition: hdf5tools.hpp:280
constexpr int h5v2_type< double >()
Definition: hdf5tools.hpp:286
H5T_NATIVE_UINT8 H5T_NATIVE_UINT16 H5T_NATIVE_UINT32 H5T_NATIVE_UINT64 constexpr int h5v2_type()
Definition: hdf5tools.hpp:278
constexpr int h5v2_type< float >()
Definition: hdf5tools.hpp:285
Here is the call graph for this function:
Here is the caller graph for this function:

◆ isDataSet()

bool Gambit::Printers::HDF5::isDataSet ( hid_t  group_id,
const std::string &  name 
)

Check if an object in a group is a dataset.

Check if an object in a file or group is a dataset.

Definition at line 616 of file hdf5tools.cpp.

References Gambit::LogTags::err, LOCAL_INFO, and Gambit::Printers::printer_error().

Referenced by Gambit::Printers::HDF5Printer2::check_consistency(), checkGroupReadable(), and Gambit::Printers::HDF5MasterBuffer::get_all_dset_names_on_disk().

617  {
618  H5O_info_t object_info;
619  herr_t err = H5Oget_info_by_name(loc_id, name.c_str(), &object_info, H5P_DEFAULT);
620  if(err<0)
621  {
622  std::ostringstream errmsg;
623  errmsg << "Attempt to check if object named '"<<name<<"' is a dataset failed! See HDF5 error for more details (stderr).";
624  printer_error().raise(LOCAL_INFO, errmsg.str());
625  }
626  return object_info.type == H5O_TYPE_DATASET;
627  }
EXPORT_SYMBOLS error & printer_error()
Printer errors.
#define LOCAL_INFO
Definition: local_info.hpp:34
Here is the call graph for this function:
Here is the caller graph for this function:

◆ lsGroup()

std::vector< std::string > Gambit::Printers::HDF5::lsGroup ( hid_t  group_id)

List object names in a group.

Definition at line 593 of file hdf5tools.cpp.

References Gambit::LogTags::err, group_ls(), LOCAL_INFO, and Gambit::Printers::printer_error().

Referenced by Gambit::Printers::HDF5Printer2::check_consistency(), checkGroupReadable(), Gambit::Printers::HDF5MasterBuffer::get_all_dset_names_on_disk(), and Gambit::Printers::lsGroup_process().

594  {
595  if(group_id<0)
596  {
597  std::ostringstream errmsg;
598  errmsg << "Error inspecting HDF5 group. The supplied group_id does not point to an open group object!";
599  printer_error().raise(LOCAL_INFO, errmsg.str());
600  }
601 
602  std::vector<std::string> out;
603  herr_t err = H5Literate(group_id, H5_INDEX_NAME, H5_ITER_NATIVE, NULL, group_ls, &out);
604 
605  if(err<0)
606  {
607  std::ostringstream errmsg;
608  errmsg << "Error encountering while iterating through HDF5 group! See HDF5 error for more details (stderr).";
609  printer_error().raise(LOCAL_INFO, errmsg.str());
610  }
611 
612  return out;
613  }
herr_t group_ls(hid_t g_id, const char *name, const H5L_info_t *, void *op_data)
Definition: hdf5tools.cpp:578
EXPORT_SYMBOLS error & printer_error()
Printer errors.
#define LOCAL_INFO
Definition: local_info.hpp:34
Here is the call graph for this function:
Here is the caller graph for this function:

◆ op_func()

herr_t Gambit::Printers::HDF5::op_func ( hid_t  loc_id,
const char *  name_in,
const H5L_info_t *  ,
void operator_data 
)
inline

Definition at line 102 of file hdf5_combine_tools.cpp.

104  {
105  //herr_t status;
106  herr_t return_val = 0;
107  H5O_info_t infobuf;
108  std::vector<std::string> &od = *static_cast<std::vector<std::string> *> (operator_data);
109  std::string name(name_in);
110 
111  //status = H5Oget_info_by_name (loc_id, name.c_str(), &infobuf, H5P_DEFAULT);
112  H5Oget_info_by_name (loc_id, name.c_str(), &infobuf, H5P_DEFAULT);
113 
114  switch (infobuf.type)
115  {
116  case H5O_TYPE_GROUP:
117  {
118  //printf ("Group: %s {\n", name);
119  //od.push_back(name);
120  break;
121 
122  }
123  case H5O_TYPE_DATASET:
124  {
125  //printf ("Dataset: %s\n", name);
126  std::string str(name);
127  if (name.find("_isvalid") == std::string::npos)
128  od.push_back(std::string(name));
129  break;
130  }
131  case H5O_TYPE_NAMED_DATATYPE:
132  //printf ("Datatype: %s\n", name);
133  break;
134  default:
135  break;
136  //printf ( "Unknown: %s\n", name);
137  }
138 
139  return return_val;
140  }
std::string str
Shorthand for a standard string.
Definition: Analysis.hpp:35

◆ op_func_aux()

herr_t Gambit::Printers::HDF5::op_func_aux ( hid_t  loc_id,
const char *  name_in,
const H5L_info_t *  ,
void operator_data 
)
inline

Definition at line 142 of file hdf5_combine_tools.cpp.

144  {
145  //herr_t status;
146  herr_t return_val = 0;
147  H5O_info_t infobuf;
148  std::vector<std::string> &od = *static_cast<std::vector<std::string> *> (operator_data);
149  std::string name(name_in);
150 
151  //status = H5Oget_info_by_name (loc_id, name.c_str(), &infobuf, H5P_DEFAULT);
152  H5Oget_info_by_name (loc_id, name.c_str(), &infobuf, H5P_DEFAULT);
153 
154  switch (infobuf.type)
155  {
156  case H5O_TYPE_GROUP:
157  {
158  //printf ("Group: %s {\n", name);
159  //od.push_back(name);
160  break;
161 
162  }
163  case H5O_TYPE_DATASET:
164  {
165  //printf ("Dataset: %s\n", name);
166  std::string str(name);
167  if (name.find("_isvalid") == std::string::npos && name != "RA_pointID" && name != "RA_MPIrank")
168  od.push_back(std::string(name));
169  break;
170  }
171  case H5O_TYPE_NAMED_DATATYPE:
172  //printf ("Datatype: %s\n", name);
173  break;
174  default:
175  break;
176  //printf ( "Unknown: %s\n", name);
177  }
178 
179  return return_val;
180  }
std::string str
Shorthand for a standard string.
Definition: Analysis.hpp:35

◆ openDataset()

hid_t Gambit::Printers::HDF5::openDataset ( hid_t  group_id,
const std::string &  name,
bool  error_off = false 
)

Dataset and dataspace manipulation.

Dataset manipulations.

Open dataset

Definition at line 691 of file hdf5tools.cpp.

References closeDataset(), hid_t, LOCAL_INFO, Gambit::Printers::printer_error(), and SIMPLE_CALL().

Referenced by checkDatasetReadable(), checkGroupReadable(), Gambit::Printers::HDF5::hdf5_stuff::Enter_Aux_Parameters(), getH5DatasetType(), and Gambit::Printers::HDF5::hdf5_stuff::hdf5_stuff().

692  {
693  hid_t dset_id;
694 
695  if(group_id < 0)
696  {
697  std::ostringstream errmsg;
698  errmsg << "Error opening HDF5 dataset '"<<name<<"'. The supplied group_id in which the dataset should be located does not point to a successfully opened group!";
699  printer_error().raise(LOCAL_INFO, errmsg.str());
700  }
701 
702  dset_id = H5Dopen2(group_id, name.c_str(), H5P_DEFAULT);
703  if(dset_id<0 and not error_off)
704  {
705  std::ostringstream errmsg;
706  errmsg << "Error opening HDF5 dataset '"<<name<<"'. Dataset may not exist at the specified location.";
707  printer_error().raise(LOCAL_INFO, errmsg.str());
708  }
709  return dset_id;
710  }
EXPORT_SYMBOLS error & printer_error()
Printer errors.
#define LOCAL_INFO
Definition: local_info.hpp:34
Here is the call graph for this function:
Here is the caller graph for this function:

◆ openFile() [1/2]

hid_t Gambit::Printers::HDF5::openFile ( const std::string &  fname,
bool  overwrite,
bool &  oldfile,
const char  access_type = 'r' 
)

File and group manipulation.

Create or open hdf5 file third argument "oldfile" is used to report whether an existing file was opened (true if yes)

Create or open hdf5 file

Definition at line 182 of file hdf5tools.cpp.

References Gambit::EOM, errorsOff(), errorsOn(), H5P_GAMBIT(), hid_t, Gambit::LogTags::info, LOCAL_INFO, Gambit::logger(), Gambit::Printers::printer_error(), Gambit::LogTags::utils, and Gambit::LogTags::warn.

Referenced by Gambit::Printers::BOOST_PP_SEQ_FOR_EACH_I(), Gambit::Printers::HDF5Printer2::check_consistency(), Gambit::Printers::HDF5Printer::common_constructor(), Gambit::Printers::HDF5::hdf5_stuff::Enter_Aux_Parameters(), Gambit::Printers::HDF5::hdf5_stuff::hdf5_stuff(), Gambit::Printers::HDF5Printer2::HDF5Printer2(), Gambit::Printers::HDF5MasterBuffer::lock_and_open_file(), and openFile().

183  {
184  //Debug
185  //std::cerr<<"Attempting to open file "<<fname<<" in mode "<<access_type<<" (overwrite="<<overwrite<<")"<<std::endl;
186 
187  hid_t file_id; // file handle
188 
189  unsigned int atype=0;
190  switch(access_type)
191  {
192  case 'r':
193  atype = H5F_ACC_RDONLY;
194  break;
195  case 'w':
196  // We let 'w' mean read/write here
197  atype = H5F_ACC_RDWR;
198  break;
199  default:
200  std::ostringstream errmsg;
201  errmsg << "Unrecognised access mode requested while trying to open HDF5 file! Saw '"<<access_type<<"'; only 'r' (read-only) and 'w' (read/wrtie) are valid. File was ("<<fname<<")";
202  printer_error().raise(LOCAL_INFO, errmsg.str());
203  break;
204  }
205 
206  if(overwrite)
207  {
208  // DANGER! Deletes existing file
209  if( remove(fname.c_str()) != 0 )
210  {
211  // Error deleting file, but probably it just didn't exist to delete
212  logger()<<LogTags::utils<<LogTags::warn<<"Failed to delete file '"<<fname<<"'! Maybe it didn't exist in the first place."<<EOM;
213  }
214  else// else deleted file with no problem
215  {
216  logger()<<LogTags::utils<<LogTags::info<<"Deleted pre-existing file "<<fname<<" (because overwrite=true)"<<EOM;
217  }
218  }
219 
220  errorsOff();
221  file_id = H5Fopen(fname.c_str(), atype, H5P_GAMBIT);
222  errorsOn();
223  if(file_id < 0)
224  {
225  if(access_type=='w')
226  {
227  /* Ok maybe file doesn't exist yet, try creating it */
228  errorsOff();
229  file_id = H5Fcreate(fname.c_str(), H5F_ACC_EXCL, H5P_DEFAULT, H5P_GAMBIT);
230  errorsOn();
231  if(file_id < 0)
232  {
233  /* Still no good; error */
234  std::ostringstream errmsg;
235  errmsg << "Failed to open existing HDF5 file, then failed to create new one! ("<<fname<<"). The file may exist but be unreadable. You can check this by trying to inspect it with the 'h5ls' command line tool.";
236  printer_error().raise(LOCAL_INFO, errmsg.str());
237  }
238  else
239  {
240  /* successfully created new file */
241  oldfile = false;
242  }
243  }
244  else
245  {
246  // Doesn't make sense to create new file if we wanted read-only mode. Error.
247  std::ostringstream errmsg;
248  errmsg << "Failed to open existing HDF5 file, and did not create new one since read-only access was specified. ("<<fname<<")";
249  printer_error().raise(LOCAL_INFO, errmsg.str());
250  }
251  }
252  else
253  {
254  /* successfully opened existing file */
255  oldfile = true;
256  }
257 
258  // DEBUG
259  #ifdef HDF5_DEBUG
260  std::cout<<"Opened file "<<fname<<" in mode "<<access_type<<", and assigned it ID "<<file_id<<std::endl;
261  #endif
262 
263  /* Return the file handle */
264  return file_id;
265  }
EXPORT_SYMBOLS error & printer_error()
Printer errors.
#define LOCAL_INFO
Definition: local_info.hpp:34
const hid_t H5P_GAMBIT(create_GAMBIT_fapl())
Const global for the GAMBIT fapl.
void errorsOff()
Silence error report (e.g. while probing for file existence)
Definition: hdf5tools.cpp:671
const Logging::endofmessage EOM
Explicit const instance of the end of message struct in Gambit namespace.
Definition: logger.hpp:99
Logging::LogMaster & logger()
Function to retrieve a reference to the Gambit global log object.
Definition: logger.cpp:95
void errorsOn()
Restore error report.
Definition: hdf5tools.cpp:681
Here is the call graph for this function:
Here is the caller graph for this function:

◆ openFile() [2/2]

hid_t Gambit::Printers::HDF5::openFile ( const std::string &  fname,
bool  overwrite = false,
const char  access_type = 'r' 
)

Create or open hdf5 file (ignoring feedback regarding whether file already existed)

Definition at line 174 of file hdf5tools.cpp.

References openFile().

175  {
176  bool tmp;
177  return openFile(fname,overwrite,tmp,access_type);
178  }
hid_t openFile(const std::string &fname, bool overwrite, bool &oldfile, const char access_type='r')
File and group manipulation.
Definition: hdf5tools.cpp:182
Here is the call graph for this function:

◆ openGroup()

hid_t Gambit::Printers::HDF5::openGroup ( hid_t  file_id,
const std::string &  name,
bool  nocreate = false 
)

Definition at line 512 of file hdf5tools.cpp.

References Gambit::LogTags::err, errorsOff(), errorsOn(), H5Gclose, hid_t, LOCAL_INFO, and Gambit::Printers::printer_error().

Referenced by Gambit::Printers::HDF5Printer2::check_consistency(), checkGroupReadable(), Gambit::Printers::HDF5Printer::common_constructor(), Gambit::Printers::HDF5::hdf5_stuff::Enter_Aux_Parameters(), Gambit::Printers::HDF5::hdf5_stuff::hdf5_stuff(), and Gambit::Printers::HDF5MasterBuffer::lock_and_open_file().

513  {
514  hid_t group_id;
515 
516  if(file_id < 0)
517  {
518  std::ostringstream errmsg;
519  errmsg << "Error opening HDF5 group '"<<name<<"'. The supplied file_id does not point to a successfully opened file!";
520  printer_error().raise(LOCAL_INFO, errmsg.str());
521  }
522 
523  // User does not want to create group
524  if(nocreate) //accessmode & H5Utils::DONOTCREATE)
525  {
526  group_id = H5Gopen2(file_id, name.c_str(), H5P_DEFAULT);
527  if(group_id<0)
528  {
529  std::ostringstream errmsg;
530  errmsg << "Error opening HDF5 group '"<<name<<"'. Group (probably) does not exist, and 'nocreate' flag is set to 'true', so we will not attempt to create one";
531  printer_error().raise(LOCAL_INFO, errmsg.str());
532  }
533  }
534  else
535  {
536  // Possibly create group and parent groups
537  std::stringstream ss(name);
538  std::stringstream path;
539  std::string gp_name;
540  while(std::getline(ss, gp_name, '/'))
541  {
542  path << "/" << gp_name;
543  errorsOff();
544  group_id = H5Gopen2(file_id, path.str().c_str(), H5P_DEFAULT);
545  errorsOn();
546  if(group_id<0)
547  {
548  /* doesn't exist; try to create it */
549  group_id = H5Gcreate2(file_id, path.str().c_str(), H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
550  if(group_id<0)
551  {
552  std::ostringstream errmsg;
553  errmsg << "Error while recursively creating/opening group '"<<name<<"'. Failed to create group '"<<path.str()<<"'";
554  printer_error().raise(LOCAL_INFO, errmsg.str());
555  }
556  }
557  herr_t err = H5Gclose(group_id);
558  if(err<0)
559  {
560  std::ostringstream errmsg;
561  errmsg << "Error closing group '"<<name<<"'!";
562  printer_error().raise(LOCAL_INFO, errmsg.str());
563  }
564  }
565  // Should exist now; open the group and return the handle
566  group_id = H5Gopen2(file_id, name.c_str(), H5P_DEFAULT);
567  if(group_id<0)
568  {
569  std::ostringstream errmsg;
570  errmsg << "Error opening HDF5 group '"<<name<<"' after recursive creation supposedly succeeded! There must be a bug in this routine, please fix.";
571  printer_error().raise(LOCAL_INFO, errmsg.str());
572  }
573  }
574  return group_id;
575  }
EXPORT_SYMBOLS error & printer_error()
Printer errors.
#define LOCAL_INFO
Definition: local_info.hpp:34
void errorsOff()
Silence error report (e.g. while probing for file existence)
Definition: hdf5tools.cpp:671
void errorsOn()
Restore error report.
Definition: hdf5tools.cpp:681
Here is the call graph for this function:
Here is the caller graph for this function:

◆ selectChunk()

std::pair< hid_t, hid_t > Gambit::Printers::HDF5::selectChunk ( const hid_t  dset_id,
std::size_t  offset,
std::size_t  length 
)

Select a simple hyperslab in a 1D dataset.

Definition at line 733 of file hdf5tools.cpp.

References combine_hdf5::dset_length, getSimpleExtentNpoints, getSpace, hid_t, LOCAL_INFO, and Gambit::Printers::printer_error().

Referenced by checkGroupReadable(), getChunk(), and Gambit::Printers::HDF5::copy_hdf5::run().

734  {
735  // Open dataspace
736  hid_t dspace_id = getSpace(dset_id);
737 
738  // Make sure that the requested chunk lies within the dataset extents
739  size_t dset_length = getSimpleExtentNpoints(dspace_id);
740 
741  if(offset + length > dset_length)
742  {
743  std::ostringstream errmsg;
744  errmsg << "Error selecting chunk from dataset in HDF5 file. Tried to select a hyperslab which extends beyond the dataset extents:" << std::endl;
745  errmsg << " offset = " << offset << std::endl;
746  errmsg << " offset+length = " << length << std::endl;
747  errmsg << " dset_length = "<< dset_length << std::endl;
748  printer_error().raise(LOCAL_INFO, errmsg.str());
749  }
750 
751  // Select a hyperslab.
752  static const size_t DSETRANK(1); // assuming 1D dataset
753  hsize_t offsets[DSETRANK];
754  offsets[0] = offset;
755  hsize_t selection_dims[DSETRANK]; // Set same as output chunks, but may have a different length
756  selection_dims[0] = length; // Adjust chunk length to input specification
757 
758  herr_t err_hs = H5Sselect_hyperslab(dspace_id, H5S_SELECT_SET, offsets, NULL, selection_dims, NULL);
759  if(err_hs<0)
760  {
761  std::ostringstream errmsg;
762  errmsg << "Error selecting chunk from dataset (offset="<<offset<<", length="<<selection_dims[0]<<") in HDF5 file. H5Sselect_hyperslab failed." << std::endl;
763  printer_error().raise(LOCAL_INFO, errmsg.str());
764  }
765 
766  // Define memory space
767  hid_t memspace_id = H5Screate_simple(DSETRANK, selection_dims, NULL);
768 
769  #ifdef HDF5_DEBUG
770  std::cout << "Debug variables:" << std::endl
771  << " dsetdims()[0] = " << this->dsetdims()[0] << std::endl
772  << " offsets[0] = " << offsets[0] << std::endl
773  << " CHUNKLENGTH = " << CHUNKLENGTH << std::endl
774  << " selection_dims[0] = " << selection_dims[0] << std::endl;
775  #endif
776 
777  return std::make_pair(memspace_id, dspace_id); // Be sure to close these identifiers after using them!
778  }
EXPORT_SYMBOLS error & printer_error()
Printer errors.
#define LOCAL_INFO
Definition: local_info.hpp:34
dataset getSimpleExtentNpoints
Definition: hdf5tools.cpp:720
Here is the call graph for this function:
Here is the caller graph for this function:

◆ setup_hdf5_points()

void Gambit::Printers::HDF5::setup_hdf5_points ( hid_t  new_group,
hid_t  type,
hid_t  type2,
unsigned long long  size_tot,
const std::string &  name 
)
inline

Definition at line 182 of file hdf5_combine_tools.cpp.

References closeDataset(), closeSpace(), dataspace, hid_t, LOCAL_INFO, Gambit::Printers::printer_error(), and Gambit::Printers::HDF5::hdf5_stuff::size_tot.

Referenced by Gambit::Printers::HDF5::hdf5_stuff::Enter_Aux_Parameters().

183  {
184  #ifdef COMBINE_DEBUG
185  std::cerr << " Creating dataset '"<<name<<"'" << std::endl;
186  #endif
187 
188  hsize_t dimsf[1];
189  dimsf[0] = size_tot;
190  hid_t dataspace = H5Screate_simple(1, dimsf, NULL);
191  if(dataspace < 0)
192  {
193  std::ostringstream errmsg;
194  errmsg<<"Failed to set up HDF5 points for copying. H5Screate_simple failed for dataset ("<<name<<").";
195  printer_error().raise(LOCAL_INFO, errmsg.str());
196  }
197  hid_t dataset_out = H5Dcreate2(new_group, name.c_str(), type, dataspace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
198  if(dataset_out < 0)
199  {
200  std::ostringstream errmsg;
201  errmsg<<"Failed to set up HDF5 points for copying. H5Dcreate2 failed for dataset ("<<name<<").";
202  printer_error().raise(LOCAL_INFO, errmsg.str());
203  }
204  hid_t dataspace2 = H5Screate_simple(1, dimsf, NULL);
205  if(dataspace2 < 0)
206  {
207  std::ostringstream errmsg;
208  errmsg<<"Failed to set up HDF5 points for copying. H5Screate_simple failed for dataset ("<<name<<"_isvalid).";
209  printer_error().raise(LOCAL_INFO, errmsg.str());
210  }
211  hid_t dataset2_out = H5Dcreate2(new_group, (name + "_isvalid").c_str(), type2, dataspace2, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
212  if(dataset2_out < 0)
213  {
214  std::ostringstream errmsg;
215  errmsg<<"Failed to set up HDF5 points for copying. H5Dcreate2 failed for dataset ("<<name<<"_isvalid).";
216  printer_error().raise(LOCAL_INFO, errmsg.str());
217  }
218 
219  // We are just going to close the newly created datasets, and reopen them as needed.
220  HDF5::closeSpace(dataspace);
221  HDF5::closeSpace(dataspace2);
222  HDF5::closeDataset(dataset_out);
223  HDF5::closeDataset(dataset2_out);
224  }
EXPORT_SYMBOLS error & printer_error()
Printer errors.
#define LOCAL_INFO
Definition: local_info.hpp:34
hid_t closeSpace(hid_t space_id)
Close dataspace.
hid_t closeDataset(hid_t dset_id)
Close dataset.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ SIMPLE_CALL() [1/3]

Gambit::Printers::HDF5::SIMPLE_CALL ( hid_t  ,
closeType  ,
hid_t  ,
H5Tclose  ,
"close"  ,
"type ID"  ,
"type ID"   
)

Close hdf5 type ID.

Close hdf5 group

Referenced by getH5DatasetType(), and openDataset().

Here is the caller graph for this function:

◆ SIMPLE_CALL() [2/3]

Gambit::Printers::HDF5::SIMPLE_CALL ( hid_t  ,
closeDataset  ,
hid_t  ,
H5Dclose  ,
"close"  ,
"dataset"  ,
"dataset"   
)

Close dataset.

Open/close dataspace; input dataset, output dataspace

◆ SIMPLE_CALL() [3/3]

dataset Gambit::Printers::HDF5::SIMPLE_CALL ( hid_t  ,
closeSpace  ,
hid_t  ,
H5Sclose  ,
"close"  ,
"dataspace"  ,
"dataspace"   
)

Get simple dataspace extent (number of points); input dataspace, output data extent (size)

◆ type_ret()

template<typename T >
T Gambit::Printers::HDF5::type_ret ( )
inline

Definition at line 42 of file hdf5_combine_tools.hpp.

References Enter_HDF5(), and hid_t.

42 {return T();}
Here is the call graph for this function:

Variable Documentation

◆ close

Gambit::Printers::HDF5::close

Definition at line 655 of file hdf5tools.cpp.

◆ closeGroup

◆ dataspace

◆ get

dataset Gambit::Printers::HDF5::get

◆ getSimpleExtentNpoints

◆ getSpace

◆ group

Gambit::Printers::HDF5::group

Definition at line 655 of file hdf5tools.cpp.

Referenced by checkGroupReadable().

◆ H5Dget_space

◆ H5Gclose

Gambit::Printers::HDF5::H5Gclose

Definition at line 655 of file hdf5tools.cpp.

Referenced by checkGroupReadable(), and openGroup().

◆ H5Sget_simple_extent_npoints

◆ hid_t

dataset Gambit::Printers::HDF5::hid_t

Definition at line 655 of file hdf5tools.cpp.

Referenced by _checkDatasetReadable_helper(), Gambit::Printers::BOOST_PP_SEQ_FOR_EACH_I(), Gambit::Printers::HDF5Printer2::check_consistency(), checkDatasetReadable(), checkFileReadable(), checkGroupReadable(), Gambit::Printers::HDF5Printer::clear_previous_points(), closeFile(), combine_hdf5_files(), Gambit::Printers::HDF5DataSet< uint >::create_dataset(), create_GAMBIT_fapl(), Gambit::ColliderBit::Perf_Plot::createFile(), createFile(), createGroup(), Gambit::Printers::DataSetInterfaceBase< uint, 0, CHUNKLENGTH >::dsetdims(), Gambit::Printers::HDF5::hdf5_stuff::Enter_Aux_Parameters(), Enter_HDF5(), Gambit::Printers::HDF5MasterBuffer::get_all_dset_names_on_disk(), Gambit::Printers::H5P_LocalBufferManager< BT(int)>::get_buffer(), Gambit::Printers::DataSetInterfaceScalar< double, CHUNKLENGTH >::get_chunk(), Gambit::Printers::HDF5DataSet< uint >::get_chunk(), Gambit::Printers::HDF5Reader::get_type(), getChunk(), getH5DatasetType(), getType(), Gambit::Printers::HDF5::hdf5_stuff::hdf5_stuff(), Gambit::Printers::HDF5Printer2::HDF5Printer2(), Gambit::Printers::HDF5DataSetBase::open_dataset(), openDataset(), openFile(), openGroup(), Gambit::Printers::HDF5DataSet< uint >::reset(), Gambit::Printers::HDF5::read_hdf5::run(), Gambit::Printers::HDF5::copy_hdf5::run(), Gambit::Printers::HDF5::ra_copy_hdf5::run(), Gambit::Printers::HDF5MasterBuffer::schedule_print(), Gambit::Printers::DataSetInterfaceScalar< double, CHUNKLENGTH >::select_chunk(), Gambit::Printers::HDF5DataSetBase::select_hyperslab(), selectChunk(), setup_hdf5_points(), Gambit::Printers::get_hdf5_data_type< char >::type(), Gambit::Printers::get_hdf5_data_type< short >::type(), Gambit::Printers::get_hdf5_data_type< int >::type(), Gambit::Printers::get_hdf5_data_type< long >::type(), Gambit::Printers::get_hdf5_data_type< long long >::type(), Gambit::Printers::get_hdf5_data_type< unsigned char >::type(), Gambit::Printers::get_hdf5_data_type< unsigned short >::type(), Gambit::Printers::get_hdf5_data_type< unsigned int >::type(), Gambit::Printers::get_hdf5_data_type< unsigned long >::type(), Gambit::Printers::get_hdf5_data_type< unsigned long long >::type(), Gambit::Printers::get_hdf5_data_type< float >::type(), Gambit::Printers::get_hdf5_data_type< double >::type(), Gambit::Printers::get_hdf5_data_type< long double >::type(), Gambit::Printers::get_hdf5_data_type< bool >::type(), type_ret(), Gambit::Printers::VertexBufferNumeric1D_HDF5< T, CHUNKLENGTH >::VertexBufferNumeric1D_HDF5(), Gambit::Printers::HDF5DataSet< uint >::write_buffer(), Gambit::Printers::HDF5DataSet< uint >::write_RA_buffer(), Gambit::Printers::H5P_LocalReadBufferManager< uint >::~H5P_LocalReadBufferManager(), and Gambit::Printers::HDF5BufferBase::~HDF5BufferBase().

◆ old_client_data

void* Gambit::Printers::HDF5::old_client_data

Definition at line 659 of file hdf5tools.cpp.

Referenced by errorsOn().

◆ old_func

group H5E_auto2_t Gambit::Printers::HDF5::old_func

global error variables (handler)

Definition at line 655 of file hdf5tools.cpp.

Referenced by errorsOn().

◆ simple_extent_npoints

dataset Gambit::Printers::HDF5::simple_extent_npoints

Definition at line 720 of file hdf5tools.cpp.