gambit is hosted by Hepforge, IPPP Durham
GAMBIT  v1.5.0-2191-ga4742ac
a Global And Modular Bsm Inference Tool
signal_handling.cpp
Go to the documentation of this file.
1 // GAMBIT: Global and Modular BSM Inference Tool
2 // *********************************************
19 
20 #include <iostream>
21 #include <signal.h>
22 #include <omp.h>
23 #include <time.h> // For nanosleep (posix only)
24 #include <cmath>
27 #include "gambit/Logs/logger.hpp"
28 #include "yaml-cpp/yaml.h"
29 
30 //#define SIGNAL_DEBUG // comment out when not debugging.
31 
32 namespace Gambit
33 {
35  std::string signal_name(int sig)
36  {
37  std::string name;
38  switch(sig){
39  case SIGINT: name="SIGINT"; break;
40  case SIGTERM: name="SIGTERM"; break;
41  case SIGUSR1: name="SIGUSR1"; break;
42  case SIGUSR2: name="SIGUSR2"; break;
43  default: name="<unlisted>";
44  }
45  return name;
46  }
47 
48  #ifdef WITH_MPI
49  ShutdownMsg::ShutdownMsg(const int code, const std::string& name)
51  : mpisize(0)
52  , myrank(0)
53  , mycode(code)
54  , name(name)
55  , buffers()
56  , buffer_status()
57  , req()
58  , comm(NULL)
59  {}
60 
61  // Default constructor
62  ShutdownMsg::ShutdownMsg()
63  : mpisize(0)
64  , myrank(0)
65  , mycode(0)
66  , name()
67  , buffers()
68  , buffer_status()
69  , req()
70  , comm(NULL)
71  {}
72 
73  void ShutdownMsg::setComm(GMPI::Comm* const c)
74  {
75  comm = c;
76  mpisize = comm->Get_size();
77  myrank = comm->Get_rank();
78  for(int i=0; i<mpisize; i++)
79  {
80  buffers.push_back(mycode);
81  buffer_status.push_back(0);
82  req.push_back(MPI_Request());
83  }
84  }
85 
86  // Send this code to all processes (non-blocking)
87  void ShutdownMsg::ISendToAll()
88  {
89  for(int i=0; i<mpisize; i++)
90  {
91  if(i!=myrank)
92  {
93  if(buffer_status[i]!=0)
94  {
95  // Buffer already in use! Error!
96  std::ostringstream errmsg;
97  errmsg<<"Tried to broadcast code "<<name<<" to all processes, but the send buffer for process "<<i<<" was already in use! This is a bug in the message send logic of the code using this object, please file a bug report."<<std::endl;
98  utils_error().raise(LOCAL_INFO, errmsg.str());
99  }
100  comm->Isend(&buffers[i], 1, i, comm->mytag, &req[i]);
101  buffer_status[i] = 1;
102  }
103  }
104  }
105 
106  // Ensure all processes have received this message (completes the send; must follow ISendToAll at some point)
107  void ShutdownMsg::Wait()
108  {
109  for(int i=0; i<mpisize; i++)
110  {
111  if(i!=myrank and buffer_status[i]==1)
112  {
113  comm->Wait(&req[i]);
114  buffer_status[i] = 0;
115  }
116  }
117  }
118 
120  std::string SignalData::shutdown_name(int code)
121  {
122  std::string name;
123  switch(code){
124  case SOFT_SHUTDOWN: name="SOFT_SHUTDOWN"; break;
125  case EMERGENCY_SHUTDOWN: name="EMERGENCY_SHUTDOWN"; break;
126  case NO_MORE_MESSAGES: name="NO_MORE_MESSAGES"; break;
127  default: name="<invalid shutdown code>"; break;
128  }
129  return name;
130  }
131 
133  constexpr int SignalData::SOFT_SHUTDOWN;
134  constexpr int SignalData::EMERGENCY_SHUTDOWN;
135  constexpr int SignalData::NO_MORE_MESSAGES;
136 
137  #endif
138 
140 
143  : jumppoint_set(false)
144  , havejumped(1) // set to zero after jump point set
145  , cleanup_function_set(false)
146  , rank(-1)
147  , shutdownBegun(0)
148  , emergency(0)
149  , POSIX_signal_noticed(false)
151  , shutdown_attempts(0)
152  , attempts_since_ff(0)
153  , ff_loop_count(0)
154  , ff_on(false)
155  , ff_count(0)
156  , inside_omp_block(false)
157  , N_signals(0)
158  #ifdef WITH_MPI
159  , MPIsize(1)
160  , _comm_rdy(false)
161  , shutdown_broadcast_done(false)
162  , looptimes(1000)
163  , timeout(500)
164  , msgs()
165  #endif
166  {
167  #ifdef WITH_MPI
168  msgs[SOFT_SHUTDOWN] = ShutdownMsg(SOFT_SHUTDOWN,shutdown_name(SOFT_SHUTDOWN));
169  msgs[EMERGENCY_SHUTDOWN] = ShutdownMsg(EMERGENCY_SHUTDOWN,shutdown_name(EMERGENCY_SHUTDOWN));
170  msgs[NO_MORE_MESSAGES] = ShutdownMsg(NO_MORE_MESSAGES,shutdown_name(NO_MORE_MESSAGES));
171  #endif
172  }
173 
175  std::string SignalData::myrank()
176  {
177  std::ostringstream tmp;
178  #ifdef WITH_MPI
179  if(rank==-1) { tmp << "UNKNOWN"; }
180  else { tmp << rank; }
181  #else
182  tmp << "(MPI DISABLED)";
183  #endif
184  return tmp.str();
185  }
186 
189  {
190  havejumped = setjmp(env);
191  jumppoint_set = true;
192  }
193 
196  {
197  cleanup = f;
198  cleanup_function_set = true;
199  }
200 
203  {
205  // Do nothing if no function has been set;
206  }
207 
210  {
211  //std::cerr << " Adding signal " << sig << std::endl; // debugging
212  if(N_signals<MAX_SIGNALS) {
214  N_signals+=1;
215  }
216  else
217  {
218  // no more space to record signals.
219  N_signals+=1;
220  }
221  }
222 
225  {
226  std::ostringstream myout;
227  if(N_signals > 0 )
228  {
229  myout << "Caught the following signals (in order):" << std::endl;
230  for(int i=0; i<N_signals; i++)
231  {
232  if(i<MAX_SIGNALS)
233  {
234  myout <<"#"<<i+1<<": Signal "<<received_signals[i]<<" ("<<signal_name(received_signals[i])<<")"<<std::endl;
235  }
236  }
237  if( (N_signals - MAX_SIGNALS) > 0)
238  {
239  myout << "Another " << (N_signals - MAX_SIGNALS) <<" signals were caught but their values were not recorded (buffer exceeded)"<<std::endl;
240  }
241  }
242  return myout.str();
243  }
244 
246  void SignalData::set_shutdown_begun(const sig_atomic_t emergnc)
247  {
248  shutdownBegun = 1;
249  emergency = emergnc;
250  // // if(ignore_signals_during_shutdown)
251  // {
252  // /// Redirect all future signals (except of course kill etc.) to the null handlers
253  // signal(SIGTERM, sighandler_null);
254  // signal(SIGINT, sighandler_null);
255  // signal(SIGUSR1, sighandler_null);
256  // signal(SIGUSR2, sighandler_null);
257  // }
258  }
259 
262  //bool SignalData::emergency_shutdown_begun() { return shutdownBegun & emergency; }
263 
266  {
267  #ifdef WITH_MPI
268  logger() << "Waiting up to "<<timeout/1000<<" seconds for all processes to sync..." << EOM;
269  // sleep setup
270  bool timedout = false;
271  std::chrono::milliseconds bar_timeout(std::lround(timeout));
272  if( signalComm->BarrierWithTimeout(bar_timeout, 9999) )
273  {
274  timedout = true; // Barrier timed out waiting for some process to enter
275  }
276  // else the barrier succeed in synchronising all processes
277  logger() << "Synchronised? " << !timedout << EOM;
278  return !timedout;
279  #else
280  return true; // Always ready if no MPI
281  #endif
282  }
283 
285  {
286  const int max_attempts=-1; // Number of extra likelihood evaluations allowed for sync attempts before we declare failure. -1 means "unlimited"
287  const int attempts_before_ff=10; // Number of times to attempt synchronisation before entering a "fast forward" period
288  const int ff_loops=1000; // Number of "fast-forward" loops to perform in a fast-forward period
289 
291  static std::chrono::time_point<std::chrono::system_clock> start(std::chrono::system_clock::now());
292 
293  if(shutdown_attempts==0)
294  {
297  logger() << "Beginning GAMBIT soft shutdown procedure. Control will be returned to the scanner plugin so "
298  << "that it can get its affairs in order in preparation for shutdown (it may cease iterating if "
299  << "it has that capability), and next iteration we will attempt to synchronise all processes and "
300  << "shut them down. If sync fails, we will loop up to "<<max_attempts<<" times (-1 means infinite), attempting to "
301  << "synchronise each time. If sync fails, an emergency shutdown will be attempted." << EOM;
303  }
304  else if(ff_on)
305  {
306  logger() << "Fast-forward active (loop "<<ff_loop_count<<"); no synchronisation attempted." << EOM;
307  // Fast-forward active; just increment counters and return
308  ++ff_loop_count;
309  if(ff_loop_count>=ff_loops)
310  {
311  logger() << "Fast-forward period finished (performed "<<ff_loop_count<<" fast loops)." << EOM;
312  ff_on = false;
313  ff_loop_count=0;
314  }
315  }
316  else if(attempts_since_ff>=attempts_before_ff)
317  {
318  // Enter "fast-forward" period
319  ff_on = true;
320  ++ff_count;
321  std::ostringstream msg;
322  msg << "rank "<<myrank()<<": Tried to synchronise for shutdown (attempt "<<shutdown_attempts<<") but failed. Will now fast-forward through "<<ff_loops<<" iterations in an attempt to 'unlock' possible MPI deadlocks with the scanner.";
323  std::cerr << msg.str() << std::endl;
324  logger() << msg.str() << EOM;
325  // Reset counters
327  }
328  else
329  {
330  // Normal sync attempt (no fast forward)
331  if(shutdown_attempts==1)
332  {
333  logger() << "Scanner did not shut down when given the chance; we will therefore assume responsibility for terminating the scan." << EOM;
334  }
335  logger() << "Attempting to synchronise for soft shutdown (attempt "<<shutdown_attempts<<")" << EOM;
336  if (all_processes_ready())
337  {
338  logger() << "Calling cleanup routines" << EOM;
339  call_cleanup();
340  std::ostringstream msg;
341  logger() << "Throwing soft shutdown exception" << EOM;
342  throw SoftShutdownException(msg.str());
343  }
344 
345  // Compute elapsed time since shutdown began
346  std::chrono::time_point<std::chrono::system_clock> current = std::chrono::system_clock::now();
347  std::chrono::duration<double> time_waited = current - start;
348 
349  if(max_attempts!=-1 and shutdown_attempts>=max_attempts)
350  {
351  logger() << "Failed to synchronise for soft shutdown! Attempting cleanup anyway, but cannot guarantee safety of the scan output." << EOM;
352  call_cleanup();
353  std::ostringstream msg;
354  #ifdef WITH_MPI
355  msg << "rank "<<myrank()<<": ";
356  #endif
357  msg << "Soft shutdown failed, emergency shutdown performed instead! (could not synchronise all processes after "<<shutdown_attempts
358  <<" attempts, and after waiting "<<std::chrono::duration_cast<std::chrono::seconds>(time_waited).count()
359  <<" seconds; fast-forward periods of "<<ff_loops<<" iterations were performed "<<ff_count
360  <<" times). Data handled by external scanner codes may have been left in an inconsistent state." << std::endl;
361  throw HardShutdownException(msg.str());
362  }
363  else
364  {
365  logger() << "Attempt to sync for soft shutdown failed (this was attempt "<<shutdown_attempts<<" of "<<max_attempts<<" (-1 means infinite)); "
366  <<std::chrono::duration_cast<std::chrono::seconds>(time_waited).count() <<" seconds have elapsed since "
367  <<"shutdown attempts began). Will allow evaluation to continue and attempt to sync again next iteration." << EOM;
368  }
371  }
372  }
373 
380  {
381  if(not shutdown_begun())
382  {
383  // If shutdown is not known to be in progress, check for MPI messages telling us to initiate shutdown
384  #ifdef WITH_MPI
385  #ifdef SIGNAL_DEBUG
387  logger() << LogTags::core << LogTags::info << "Doing Iprobe to check for shutdown messages from other processes (with MPI tag "<<signalComm->mytag<<")" << EOM;
388  #endif
389  if(signalComm->Iprobe(MPI_ANY_SOURCE, signalComm->mytag))
390  {
391  #ifdef SIGNAL_DEBUG
392  logger() << LogTags::core << LogTags::info << "Shutdown message detected; doing Recv" << EOM;
393  #endif
394  int code;
395  MPI_Status msg_status;
396  signalComm->Recv(&code, 1, MPI_ANY_SOURCE, signalComm->mytag, &msg_status);
397 
398  // Check what code was received and use it to determined what kind of shutdown to do
399  if(code==SOFT_SHUTDOWN)
400  {
402  logger() << LogTags::core << LogTags::info << "Received SOFT shutdown message from process with rank " << msg_status.MPI_SOURCE << EOM;
403  }
404  else if(code==EMERGENCY_SHUTDOWN)
405  {
406  set_shutdown_begun(1); // '1' argument means emergency set also.
407  logger() << LogTags::core << LogTags::info << "Received EMERGENCY shutdown message from process with rank " << msg_status.MPI_SOURCE << EOM;
408  }
409  else
410  {
411  std::ostringstream ss;
412  ss << "Received UNRECOGNISED shutdown message from process with rank " << msg_status.MPI_SOURCE<<". Performing emergency shutdown, but please note that this indicates a ***BUG*** somewhere in the signal handling code!!!";
413  std::cout << ss.str() << std::endl;
414  logger() << LogTags::core << LogTags::info << ss.str() << EOM;
415  set_shutdown_begun(1); // '1' argument means emergency set also.
416  }
417 
419  }
420  #ifdef SIGNAL_DEBUG
421  else
422  {
423  logger() << LogTags::core << LogTags::info << "No shutdown message detected; continuing as normal" << EOM;
424  }
425  #endif
426  #endif
427  }
428 
429  // Check shutdown status again (might have changed due to MPI message receipt)
430  if(shutdown_begun())
431  {
433  {
434  std::ostringstream ss;
435  ss << "Process "<<rank<<": Shutdown signal detected!"<< std::endl;
436  // std::cerr << ss.str();
437  logger() << ss.str() << display_received_signals() << EOM;
438  POSIX_signal_noticed = true;
439  }
440 
441  std::ostringstream ss;
442  static int loopi(0);
443  logger() << "Shutdown is in progress; emergency="<< emergency <<" (loop="<<loopi<<")"<< EOM;
444  ++loopi;
445  logger() << ss.str() << EOM;
446 
447  #ifdef WITH_MPI
448  if(not shutdown_due_to_MPI_message) // Don't broadcast another shutdown message if we are shutting down due to an MPI message we received. Assume that all processes will get the first message (otherwise for 1000 process job we will end up with 1000*1000 shutdown messages clogging up the network)
449  {
450  // Broadcast shutdown message to all processes
451  int shutdown_code;
452  if(emergency)
453  {
454  shutdown_code = EMERGENCY_SHUTDOWN;
455  }
456  else
457  {
458  shutdown_code = SOFT_SHUTDOWN;
459  }
460  broadcast_shutdown_signal(shutdown_code);
461  }
462  else if(emergency)
463  {
464  throw MPIShutdownException("Received emergency shutdown command via MPI! Terminating run.");
465  }
466  #endif
467 
468  }
469  return shutdown_begun();
470  }
471 
472 
474  void SignalData::update_looptime(double /*newtime*/)
475  {
476  // Leave this as 1 second now that likelihood calculation is disabled during shutdown
477  }
478 
483  #ifdef WITH_MPI
484  void SignalData::discard_excess_shutdown_messages()
485  {
487  #ifdef SIGNAL_DEBUG
488  logger() << LogTags::core << LogTags::info << "Doing Iprobe to check for shutdown signals from other processes (with MPI tag "
489  <<signalComm->mytag<<"). These will be discarded (since we are inside the 'discard_excess_shutdown_messages' routine)" << EOM;
490  #endif
491  int max_loops = 2*signalComm->Get_size(); // At most should be one message from every process (minus one), so we will check twice as many times as this before deciding that something has gone horribly wrong.
492 
493  int code;
494  signalComm->Recv_all(&code, 1, MPI_ANY_SOURCE, signalComm->mytag, max_loops);
495  }
496 
501  void SignalData::ensure_no_more_shutdown_messages()
502  {
503  int mpiSize = signalComm->Get_size();
504  int myRank = signalComm->Get_rank();
505 
506  // Recv all shutdown messages from other processes
507  logger() << LogTags::core << LogTags::debug << "Receiving all shutdown messages" << EOM;
508  for(int rank=0; rank<mpiSize; rank++)
509  {
510  if(rank!=myRank)
511  {
512  //std::cerr<<"Rank "<<myRank<<" attempting to cleanup shutdown messages from rank "<<rank<<std::endl;
513  int loop = 0;
514  bool more_messages = true;
515  while(more_messages)
516  {
517  int code;
518  //std::cerr<<"Rank "<<myRank<<": Messages waiting from rank "<<rank<<"? "<<signalComm->Iprobe(rank, signalComm->mytag)<<std::endl;
519  signalComm->Recv(&code, 1, rank, signalComm->mytag);
520  //std::cerr<<"Rank "<<myRank<<": received code "<<shutdown_name(code)<<" from rank "<<rank<<std::endl;
521  if(code==NO_MORE_MESSAGES) more_messages = false;
522  loop++;
523  if(loop>2*mpiSize)
524  {
525  // I think there should only be one shutdown message from each process, so this
526  // condition is extreme and definitely indicates a problem.
527  std::ostringstream errmsg;
528  errmsg<<"ensure_no_more_shutdown_messages function has been looping on rank "<<myRank<<" for "<<loop<<" iterations (receiving messages from rank "<<rank<<" process), but there are only "<<mpiSize<<" processes in this job. There should not be anywhere near this many shutdown messages to receive, so something has gone horribly wrong. Please report this as a bug.";
529  utils_error().raise(LOCAL_INFO, errmsg.str());
530  }
531  }
532  }
533  }
534 
535  // Ensure all shutdown messages from this process have been fully sent
536  // (not necessary for the corresponding Recv's to complete, so no deadlock problem,
537  // but needed to confirm that the messages were sent and allow MPI to clean them up
538  // properly).
539  logger()<<LogTags::core << LogTags::info<<"Cleaning up shutdown message send buffers"<<EOM;
540  for(auto it=msgs.begin(); it!=msgs.end(); ++it)
541  {
542  it->second.Wait();
543  }
544  }
545  #endif
546 
548 
551  {
552  inside_omp_block = 1;
553  // Debugging
554  //std::cerr << "rank " << rank <<": ENTERING_MULTITHREADED_REGION" << std::endl;
555  //std::cerr << "rank " << rank <<": signaldata().inside_omp_block=1 " << std::endl;
556  }
557 
560  {
561  inside_omp_block = 0;
562  // Debugging
563  //std::cerr << "rank " << rank <<": LEAVING_MULTITHREADED_REGION" << std::endl;
564  //std::cerr << "rank " << rank <<": signaldata().inside_omp_block=0 " << std::endl;
565  }
566 
569  {
570  return inside_omp_block;
571  }
572 
574 
575  #ifdef WITH_MPI
576  bool SignalData::comm_ready() { return (GMPI::Is_initialized() and _comm_rdy); }
578 
581  void SignalData::set_MPI_comm(GMPI::Comm* comm)
582  {
583  signalComm = comm;
584  _comm_rdy = true;
585  rank = comm->Get_rank();
586  MPIsize = comm->Get_size();
587  // Set communicator for shutdown message objects
588  for(auto it=msgs.begin(); it!=msgs.end(); ++it)
589  {
590  it->second.setComm(comm);
591  }
592  }
593 
595  void SignalData::broadcast_shutdown_signal(int shutdown_code)
596  {
597  if(MPIsize>1)
598  {
599  if(shutdown_code==NO_MORE_MESSAGES or not shutdown_broadcast_done)
600  {
601  if(comm_ready())
602  {
603  // Broadcast signal to all processes (might not work if something errornous is occuring)
604  #ifdef SIGNAL_DEBUG
605  logger() << LogTags::core << LogTags::info << "Broadcasting shutcode code " <<shutdown_name(shutdown_code)<< " with MPI tag "<<signalComm->mytag<< EOM;
606  #endif
607  msgs[shutdown_code].ISendToAll();
608  logger() << LogTags::core << LogTags::info << shutdown_name(shutdown_code) <<" code broadcast to all processes" << EOM;
609  }
610  else
611  {
613  std::ostringstream errmsg;
614  errmsg << "Tried to broadcast_shutdown_signal ("<<shutdown_name(shutdown_code)<<"), but MPI communicator is not ready! (either MPI is uninitialised or a communicator has not been set). This is a bug, please report it.";
615  utils_error().raise(LOCAL_INFO, errmsg.str());
616  }
617  shutdown_broadcast_done = true;
618  } // Don't need to broadcast twice (NOTE: might need to trigger change from soft to emergency shutdown?)
619  #ifdef SIGNAL_DEBUG
620  else
621  {
622  logger() << LogTags::core << LogTags::info << "Received instruction to broadcast code " <<shutdown_name(shutdown_code)
623  <<", however shutdown_broadcast_done=true is already set, so skipping the broadcast!"<< EOM;
624  }
625  #endif
626  }
627  }
628  #endif
629 
631 
634  {
635  static SignalData data;
636  #ifdef WITH_MPI
637  // If we are using MPI, it is required that the signaldata object be initialised with a communicator object
638  // This 'ifdef' block ensures that this happens, or else throws an error.
639  static int access_count = 0;
640  if(access_count==0){ access_count += 1; }
641  else if(access_count==1)
642  {
643  // Check that communicator object has been initialised
644  if(not data.comm_ready())
645  {
646  std::ostringstream errmsg;
647  errmsg << "Error retrieving global SignalData object! An MPI communicator has not been provided to this object! Please provide one via the 'set_MPI_comm' the first time that 'signaldata()' is called.";
648  utils_error().raise(LOCAL_INFO, errmsg.str());
649  }
650  }
651  #endif
652  return data;
653  }
654 
657 
659 
661 
664  void sighandler_soft(int sig)
665  {
666  // We will avoid touching streams in this "clean" shutdown mode since technically it is undefined behaviour, so no messages here.
668  signaldata().add_signal(sig); // I think this should be ok... but can delete it if there are any problems
669  }
670 
672 
675  void set_signal_handler(const YAML::Node& keyvalnode, const int sig, const std::string& def_mode)
676  {
677  std::string shutdown_mode;
678  if(keyvalnode["signal_handling"]) {
679  YAML::Node signal_options = keyvalnode["signal_handling"];
680  if(signal_options[signal_name(sig)]) {
681  shutdown_mode = signal_options[signal_name(sig)].as<std::string>();
682  }else{
683  shutdown_mode = def_mode;
684  }
685  }else{
686  shutdown_mode = def_mode;
687  }
688  logger()<< "Setting action on "<<signal_name(sig)<<" to '"<<shutdown_mode<<"'"<<EOM;
689  // if (shutdown_mode=="hard_shutdown"){ signal(sig, sighandler_hard); }
690  // else if (shutdown_mode=="emergency_shutdown"){ signal(sig, sighandler_emergency); }
691  // else if (shutdown_mode=="emergency_shutdown_longjmp"){ signal(sig, sighandler_emergency_longjmp); }
692  //else
693  if (shutdown_mode=="soft_shutdown"){ signal(sig, sighandler_soft); }
694  //else if (shutdown_mode=="null"){ signal(sig, sighandler_null); }
695  else {
696  std::ostringstream msg;
697  msg << "Invalid shutdown mode requested for signal "<<signal_name(sig)<<" ("<<sig<<")"<<" (via YAML file option '"<<signal_name(sig)<<"' in KeyValue section under 'signal_handling'). Value received was '"<<shutdown_mode<<"'. Valid shutdown modes are:" <<std::endl;
698  msg << " 'hard_shutdown' -- Exit immediately." <<std::endl;
699  msg << " 'emergency_shutdown' -- Attempt to save printer/resume data and then immediately exit." <<std::endl;
700  msg << " 'emergency_shutdown_longjmp' -- Longjmp to outside of likelihood loop, then attempt to save printer/resume data and exit"<<std::endl;
701  msg << " 'soft_shutdown' -- Safest: attempt to synchronise processes at safe location, then save printer/resume data and exit." <<std::endl;
702  msg << " 'null' -- Ignore signal. Use at own risk!" <<std::endl;
703  msg << "The default shutdown mode on signal "<<signal_name(sig)<<" is '"<<def_mode<<"'." <<std::endl;
704  std::cerr << msg.str();
705  std::cerr << std::endl;
706  exit(EXIT_FAILURE);
707  }
708  }
709 
710 }
711 
712 
greatScanData data
Definition: great.cpp:38
std::string display_received_signals()
Print to string a list of the signals received so far by this process.
void set_shutdown_begun(const sig_atomic_t emergnc=0)
Register that shutdown has begun.
void entering_multithreaded_region()
Check if shutdown is in progress and raise appropriate termination exception if so.
EXPORT_SYMBOLS error & utils_error()
Utility errors.
void call_cleanup()
Call cleanup function.
bool all_processes_ready()
Attempt to synchronise all processes, but abort if it takes too long.
jmp_buf env
Saved information on calling environment for longjmp.
#define LOCAL_INFO
Definition: local_info.hpp:34
Logging access header for GAMBIT.
Special exception used during controlled early shutdown.
Definition: exceptions.hpp:346
void attempt_soft_shutdown()
Perform soft shutdown if processes can be synchronised.
void leaving_multithreaded_region()
Exit threadsafe signal handling mode.
void add_signal(int sig)
Check if emergency shutdown is in progress.
GAMBIT signal handling functions.
EXPORT_SYMBOLS bool check_if_shutdown_begun()
Check for signals that early shutdown is required If an MPI message telling us to perform an emergenc...
std::string signal_name(int sig)
Translate signal codes to strings.
Special exception raised when emergency shutdown triggered via MPI.
Definition: exceptions.hpp:364
volatile sig_atomic_t shutdownBegun
Flag to warn if early shutdown is already in process.
Special exception used during emergency early shutdown.
Definition: exceptions.hpp:355
EXPORT_SYMBOLS SignalData & signaldata()
Retrieve global instance of signal handler options struct.
const Logging::endofmessage EOM
Explicit const instance of the end of message struct in Gambit namespace.
Definition: logger.hpp:100
EXPORT_SYMBOLS Logging::LogMaster & logger()
Function to retrieve a reference to the Gambit global log object.
Definition: logger.cpp:95
int received_signals[MAX_SIGNALS]
void setjump()
Set jump point;.
void(* void_func)()
Set cleanup function to run during emergency shutdown.
void sighandler_soft(int sig)
Signal handler functions.
void set_signal_handler(const YAML::Node &keyvalnode, const int sig, const std::string &def_mode)
Choose signal handler for a given signal via yaml file option.
A simple C++ wrapper for the MPI C bindings.
bool POSIX_signal_noticed
Flag to indicate if POSIX shutdown signal has been noticed.
SignalData()
SignalData member functions.
bool shutdown_begun()
Check if (any kind of) shutdown is in progress.
void update_looptime(double newtime)
Extra functions needed in MPI mode.
Variables for use in signal handlers.
std::string myrank()
Retrieve MPI rank as a string (for log messages etc.)
static const int MAX_SIGNALS
Array to record received signals (up to max_signals)
bool inside_multithreaded_region()
Report &#39;true&#39; if inside a multithreaded region (according to our own flag)
bool inside_omp_block
Flag to switch signal handling behavior to multithreaded mode (i.e.
TODO: see if we can use this one:
Definition: Analysis.hpp:33
void set_cleanup(void_func f)
Set cleanup function.
volatile sig_atomic_t emergency
Flag to warn if the shutdown that is in progress is an emergency shutdown (use to decided whether to ...
int shutdown_attempts
Number of times synchronisation for soft shutdown has been attempted;.