gambit is hosted by Hepforge, IPPP Durham
GAMBIT  v1.5.0-2191-ga4742ac
a Global And Modular Bsm Inference Tool
module_harvester Namespace Reference

Functions

def main (argv)
 

Variables

string toolsfile = "./Utils/scripts/harvesting_tools.py"
 

Function Documentation

◆ main()

def module_harvester.main (   argv)

Definition at line 56 of file module_harvester.py.

References harvesting_tools.addifbefunctormacro(), harvesting_tools.addiffunctormacro(), harvesting_tools.find_and_harvest_headers(), harvesting_tools.get_default_boss_namespaces(), harvesting_tools.get_type_equivalencies(), harvesting_tools.neatsplit(), Gambit::ColliderBit.print(), harvesting_tools.readlines_nocomments(), harvesting_tools.retrieve_module_type_headers(), harvesting_tools.retrieve_rollcall_headers(), harvesting_tools.update_module(), and harvesting_tools.update_only_if_different().

56 def main(argv):
57 
58  # Lists of modules to exclude; anything starting with one of these strings is excluded.
59  exclude_modules=set([])
60 
61  # Handle command line options
62  verbose = False
63  try:
64  opts, args = getopt.getopt(argv,"vx:",["verbose","exclude-modules="])
65  except getopt.GetoptError:
66  print('Usage: module_harvestor.py [flags]')
67  print(' flags:')
68  print(' -v : More verbose output')
69  print(' -x module1,module2,... : Exclude module1, module2, etc.')
70  sys.exit(2)
71  for opt, arg in opts:
72  if opt in ('-v','--verbose'):
73  verbose = True
74  print('module_harvester.py: verbose=True')
75  elif opt in ('-x','--exclude-modules'):
76  exclude_modules.update(neatsplit(",",arg))
77  exclude_header = exclude_modules
78  module_rollcall_headers=set([])
79  module_type_headers=set([])
80  full_rollcall_headers=[]
81  full_type_headers=[]
82  modules=set([])
83 
84  # List of headers to search
85  rollcall_headers = set(["gambit/Backends/backend_rollcall.hpp", "Models/include/gambit/Models/model_rollcall.hpp"])
86  type_headers = set(["gambit/Elements/types_rollcall.hpp"])
87 
88  # List of headers NOT to search (things we know are not module rollcall headers or module type headers,
89  # but are included in module_rollcall.hpp or types_rollcall.hpp)
90  exclude_header.update(["shared_types.hpp", "backend_macros.hpp", "backend_undefs.hpp", "identification.hpp",
91  "yaml.h", "module_macros_incore.hpp", "module_macros_inmodule.hpp", "module_macros_common.hpp",
92  "model_macros.hpp"])
93 
94  # List of types not to bother looking for the definitions of.
95  intrinsic_types=set(["char", "bool", "short", "int", "long", "float", "double", "std::string"])
96 
97  # List of types NOT to return (things we know are not printable, but can appear in START_FUNCTION calls)
98  exclude_types=set(["void"])
99 
100  # List of directory names to ignore when searching for headers
101  exclude_dirs=set([".git","build","doc","cmake","extras","config","contrib","runs","Logs","Printers","scratch","installed","scripts","gum"])
102 
103  # If any variation of pybind11 is in the excluded_modules list, ditch all pybind11 dependent types
104  if "pybind" in exclude_header or "pybind11" in exclude_header or "Pybind" in exclude_header or "Pybind11" in exclude_header :
105  exclude_pybind11 = True
106  else:
107  exclude_pybind11 = False
108 
109  # Load up the sets of equivalent types and namespaces
110  equiv_ns = get_default_boss_namespaces()
111  equiv_classes = get_type_equivalencies(equiv_ns)
112 
113  # Get list of rollcall header files to search
114  module_rollcall_headers.update(retrieve_rollcall_headers(verbose,".",exclude_header))
115  rollcall_headers.update(module_rollcall_headers)
116  # Get list of module type header files to search
117  module_type_headers.update(retrieve_module_type_headers(verbose,".",exclude_header))
118 
119  if verbose: print("Module rollcall headers identified:")
120  for h in module_rollcall_headers:
121  if verbose: print(' ',h)
122  h_parts = neatsplit('\/',h)
123  modules.add(h_parts[1])
124  if verbose:
125  print("Module type headers identified:")
126  for h in module_type_headers:
127  print(' ',h)
128 
129 
130  # Generate a c++ header containing all the module type headers we have just harvested.
131  towrite = "\
132 // GAMBIT: Global and Modular BSM Inference Tool\n\
133 // *********************************************\n\
134 /// \\file \n\
135 /// \n\
136 /// Compile-time registration of GAMBIT module \n\
137 /// types. \n\
138 /// \n\
139 /// This file was automatically generated by \n\
140 /// module_harvester.py. Do not modify. \n\
141 /// \n\
142 /// Do not add to this if you want to add new \n\
143 /// types associated with a new (or old) module. \n\
144 /// Just add your model type header as \n\
145 /// XBit/include/gambit/XBit/XBit_types.hpp and \n\
146 /// rest assured that module_harvester.py will \n\
147 /// make sure it ends up here. \n\
148 /// \n\
149 /// *********************************************\n\
150 /// \n\
151 /// Authors: \n\
152 /// \n\
153 /// \\author The GAMBIT Collaboration \n\
154 /// \date "+datetime.datetime.now().strftime("%I:%M%p on %B %d, %Y")+"\n\
155 /// \n\
156 /// *********************************************\n\
157  \n\
158 #ifndef __module_types_rollcall_hpp__ \n\
159 #define __module_types_rollcall_hpp__ \n\
160  \n\
161 // Automatically-generated list of module types. \n"
162 
163  for h in module_type_headers:
164  towrite+='#include \"{0}\"\n'.format(h)
165  towrite+="\n#endif // defined __module_types_rollcall_hpp__\n"
166 
167  # Don't touch any existing file unless it is actually different from what we will create
168  header = "./Elements/include/gambit/Elements/module_types_rollcall.hpp"
169  candidate = "./scratch/build_time/module_types_rollcall.hpp.candidate"
170  with open(candidate,"w") as f: f.write(towrite)
171  update_only_if_different(header, candidate)
172 
173  print("Harvesting types from headers...")
174 
175  # Recurse through chosen rollcall headers, locating all the included headers therein, and find them all
176  # in the gambit source tree so that we can parse them for types etc.
177  if verbose: print(" Searching rollcall headers...")
178  find_and_harvest_headers(rollcall_headers,full_rollcall_headers,exclude_header,exclude_dirs,verbose=verbose)
179  if verbose: print(" Searching type headers...")
180  find_and_harvest_headers(type_headers,full_type_headers,exclude_header,exclude_dirs,verbose=verbose)
181 
182  # Search through rollcall headers and look for macro calls that create module_functors or safe pointers to them
183  types=set(["ModelParameters", "double", "float", "std::vector<double>", "std::vector<float>"]) #Manually add these, as they must always be included.
184  non_module_types=set(["ModelParameters", "double", "float", "std::vector<double>", "std::vector<float>"])
185  returned_types = { "all" : types, "non_module" : non_module_types }
186  for header in full_rollcall_headers:
187  with io.open(header, encoding='utf-8') as f:
188  if verbose: print(" Scanning header {0} for types used to instantiate module functor class templates".format(header))
189  module = "__NotAModule__"
190  continued_line = ""
191  ignore_lines = False
192  nguards = 0
193  for line in readlines_nocomments(f):
194  continued_line += line
195  if line.strip().endswith(","): continue
196  # If this line defines the module name, update it.
197  module = update_module(continued_line,module)
198  # If a HAVE_PYBIND11 guard is found and pybind11 is ditched, pause reading
199  if line.strip().endswith("#ifdef HAVE_PYBIND11") and exclude_pybind11 :
200  ignore_lines = True
201  # Count the number of ifdefs or ifndefs while ignoring lines to ensure it ends where it should
202  if ignore_lines and ("ifdef" in line or "ifndef" in line) :
203  nguards += 1
204  if ignore_lines and line.strip().endswith("#endif"):
205  nguards -= 1
206  # if there is no open ifdef or ifndef then restore ignore_lines
207  if nguards == 0:
208  ignore_lines = False
209  if not ignore_lines:
210  # Check for calls to module functor creation macros, and harvest the types used.
211  addiffunctormacro(continued_line,module,modules,returned_types,full_type_headers,intrinsic_types,exclude_types,equiv_classes,equiv_ns,verbose=verbose)
212  continued_line = ""
213 
214  if verbose:
215  print("Found types for module functions:")
216  for t in types:
217  print(' ',t)
218 
219  # Search through rollcall and frontend headers and look for macro calls that create backend_functors or safe pointers to them
220  be_types=set()
221  type_packs=set()
222  for header in full_rollcall_headers:
223  with io.open(header, encoding='utf-8') as f:
224  if verbose: print(" Scanning header {0} for types used to instantiate backend functor class templates".format(header))
225  continued_line = ""
226  ignore_lines = False
227  nguards = 0
228  for line in readlines_nocomments(f):
229  continued_line += line
230  if line.strip().endswith(","): continue
231  # If a HAVE_PYBIND11 guard is found and pybind11 is ditched, pause reading
232  if line.strip().endswith("#ifdef HAVE_PYBIND11") and exclude_pybind11 :
233  ignore_lines = True
234  # Count the number of ifdefs or ifndefs while ignoring lines to ensure it ends where it should
235  if ignore_lines and ("ifdef" in line or "ifndef" in line) :
236  nguards += 1
237  if ignore_lines and line.strip().endswith("#endif"):
238  nguards -= 1
239  # if there is no open ifdef or ifndef then restore ignore_lines
240  if nguards == 0:
241  ignore_lines = False
242  if not ignore_lines:
243  # Check for calls to backend functor creation macros, and harvest the types used.
244  addifbefunctormacro(continued_line,be_types,type_packs,equiv_classes,equiv_ns,verbose=verbose)
245  continued_line = ""
246 
247  if verbose:
248  print("Found types for backend functions and variables:")
249  for t in be_types:
250  if t != "": print(' ',t)
251 
252  # Generate a c++ header containing the backend functor template specialisations, using all the backend types we have harvested.
253  towrite = "\
254 // GAMBIT: Global and Modular BSM Inference Tool\n\
255 // *********************************************\n\
256 /// \\file \n\
257 /// \n\
258 /// Automatically generated template \n\
259 /// specialisations for backend functors. \n\
260 /// \n\
261 /// This file was automatically generated by \n\
262 /// module_harvester.py. Do not modify. \n\
263 /// \n\
264 /// The content is harvested from: \n\
265 /// - rollcall headers in module_rollcall.hpp \n\
266 /// - frontend headers in backend_rollcall.hpp \n\
267 /// - types registered in types_rollcall.hpp \n\
268 /// \n\
269 /// *********************************************\n\
270 /// \n\
271 /// Authors: \n\
272 /// \n\
273 /// \\author The GAMBIT Collaboration \n\
274 /// \date "+datetime.datetime.now().strftime("%I:%M%p on %B %d, %Y")+"\n\
275 /// \n\
276 /// *********************************************\n\
277  \n\
278 #ifndef __backend_functor_types_hpp__ \n\
279 #define __backend_functor_types_hpp__ \n\
280  \n\
281 #include \"gambit/Elements/types_rollcall.hpp\" \n\
282 #include \"gambit/Elements/functor_definitions.hpp\"\n\
283  \n\
284 namespace Gambit \n\
285 { "
286  for tp in type_packs:
287  towrite+="\n\
288  template class backend_functor_common<{0}>;\n\
289  template class backend_functor<{0}>;".format(tp)+"\n"
290  towrite+="}\n\n#endif // defined __backend_functor_types_hpp__\n"
291 
292  # Don't touch any existing file unless it is actually different from what we will create
293  header = "./Backends/include/gambit/Backends/backend_functor_types.hpp"
294  candidate = "./scratch/build_time/backend_functor_types.hpp.candidate"
295  with open(candidate,"w") as f: f.write(towrite)
296  update_only_if_different(header, candidate)
297 
298 
299  # Generate a c++ source file containing the module functor template specialisations, using all the module types we have harvested.
300  towrite = "\
301 // GAMBIT: Global and Modular BSM Inference Tool\n\
302 // *********************************************\n\
303 /// \\file \n\
304 /// \n\
305 /// Automatically generated template \n\
306 /// specialisations for module functors, for \n\
307 /// the full GAMBIT executable. \n\
308 /// \n\
309 /// This file was automatically generated by \n\
310 /// module_harvester.py. Do not modify. \n\
311 /// \n\
312 /// The content is harvested from: \n\
313 /// - rollcall headers in module_rollcall.hpp \n\
314 /// - types registered in types_rollcall.hpp \n\
315 /// \n\
316 /// *********************************************\n\
317 /// \n\
318 /// Authors: \n\
319 /// \n\
320 /// \\author The GAMBIT Collaboration \n\
321 /// \date "+datetime.datetime.now().strftime("%I:%M%p on %B %d, %Y")+"\n\
322 /// \n\
323 /// *********************************************\n\
324  \n\
325 #ifndef __module_functor_types_hpp__ \n\
326 #define __module_functor_types_hpp__ \n\
327  \n\
328 #include \"gambit/Elements/types_rollcall.hpp\" \n\
329 #include \"gambit/Elements/functor_definitions.hpp\"\n\
330  \n\
331 namespace Gambit \n\
332 { \n"
333  for t in types:
334  towrite += ' template class module_functor<{0}>;\n'.format(t)
335  towrite+="}\n\n#endif // defined __module_functor_types_hpp__\n"
336 
337  # Don't touch any existing file unless it is actually different from what we will create
338  source = "./Elements/include/gambit/Elements/module_functor_types.hpp"
339  candidate = "./scratch/build_time/module_functor_types.hpp.candidate"
340  with open(candidate,"w") as f: f.write(towrite)
341  update_only_if_different(source, candidate)
342 
343  if verbose:
344  print("\nGenerated Core/module_rollcall.hpp.")
345  print("Generated Elements/module_types_rollcall.hpp.")
346  print("Generated Elements/module_functor_types.hpp.")
347  print("Generated Backends/backend_functor_types.hpp.")
348 
349  # Pickle the types for later usage by standalone_facilitator.py
350  with open('./scratch/build_time/harvested_types.pickle', 'wb') as handle:
351  pickle.dump(returned_types, handle, protocol=pickle.HIGHEST_PROTOCOL)
352 
353 # Handle command line arguments (verbosity)
void print(MixMatrix A)
Helper function to print a matrix.
def addifbefunctormacro(line, be_typeset, type_pack_set, equiv_classes, equiv_ns, verbose=False)
def retrieve_rollcall_headers(verbose, install_dir, excludes)
def find_and_harvest_headers(header_set, fullheadlist, exclude_set, dir_exclude_set, verbose=False)
def get_default_boss_namespaces()
def addiffunctormacro(line, module, all_modules, typedict, typeheaders, intrinsic_types, exclude_types, equiv_classes, equiv_ns, verbose=False)
def update_only_if_different(existing, candidate, verbose=True)
def neatsplit(regex, string)
def update_module(line, module)
def retrieve_module_type_headers(verbose, install_dir, excludes)
def get_type_equivalencies(nses)
Here is the call graph for this function:

Variable Documentation

◆ toolsfile

string module_harvester.toolsfile = "./Utils/scripts/harvesting_tools.py"

Definition at line 53 of file module_harvester.py.