gambit is hosted by Hepforge, IPPP Durham
GAMBIT  v1.5.0-2191-ga4742ac
a Global And Modular Bsm Inference Tool
harvesting_tools Namespace Reference

Functions

def get_default_boss_namespaces ()
 
def get_type_equivalencies (nses)
 
def comment_remover (text)
 
def readlines_nocomments (f)
 
def neatsplit (regex, string)
 
def excluded (string, st)
 
def sorted_nicely (l)
 
def check_for_declaration (input_snippet, module, all_modules, local_namespace, candidate_type)
 
def check_for_namespace (input_snippet, local_namespace)
 
def addifheader (line, headerset, exclude_set, verbose=False)
 
def update_module (line, module)
 
def first_simple_type_equivalent (candidate_in, equivs, nses, existing)
 
def strip_ws (s, qualifiers)
 
def addiffunctormacro (line, module, all_modules, typedict, typeheaders, intrinsic_types, exclude_types, equiv_classes, equiv_ns, verbose=False)
 
def addifbefunctormacro (line, be_typeset, type_pack_set, equiv_classes, equiv_ns, verbose=False)
 
def get_headers (path, header_set, exclude_set, verbose=False)
 
def find_and_harvest_headers (header_set, fullheadlist, exclude_set, dir_exclude_set, verbose=False)
 
def retrieve_rollcall_headers (verbose, install_dir, excludes)
 
def retrieve_module_type_headers (verbose, install_dir, excludes)
 
def get_all_files_with_ext (verbose, starting_dir, ext_set, kind)
 
def retrieve_generic_headers (verbose, starting_dir, kind, excludes, exclude_list=[])
 
def same (f1, f2)
 
def update_only_if_different (existing, candidate, verbose=True)
 
def make_module_rollcall (rollcall_headers, verbose)
 

Variables

string default_bossed_versions = "./Backends/include/gambit/Backends/default_bossed_versions.hpp"
 
string equiv_config = "./config/resolution_type_equivalency_classes.yaml"
 

Function Documentation

◆ addifbefunctormacro()

def harvesting_tools.addifbefunctormacro (   line,
  be_typeset,
  type_pack_set,
  equiv_classes,
  equiv_ns,
  verbose = False 
)

Definition at line 349 of file harvesting_tools.py.

References first_simple_type_equivalent(), neatsplit(), and strip_ws().

Referenced by module_harvester.main().

349 def addifbefunctormacro(line,be_typeset,type_pack_set,equiv_classes,equiv_ns,verbose=False):
350 
351  command_index = {"BE_VARIABLE":2,
352  "BE_FUNCTION":2,
353  "BE_CONV_FUNCTION":2,
354  "BACKEND_REQ":0,
355  "BACKEND_REQ_FROM_GROUP":0}
356 
357  line = re.sub(";", "", line)
358  splitline = neatsplit('\(|\)|,|\s',line)
359 
360  qualifier_list = ["const", "struct"]
361 
362  if len(splitline)>1 and splitline[0] in command_index.keys():
363  #This line defines a backend functor and one or more of the arguments defines a candidate type
364 
365  if splitline[0].startswith("BACKEND_REQ"):
366  args = re.sub("\s*BACKEND_REQ(_FROM_GROUP)?\s*\(.*?,\s*\(.*?\)\s*,\s*", "", re.sub("\s*\)\s*$", "", line) )
367  args = args.strip()
368  if re.search("\)\s*\)\s*$", line):
369  #This is a backend function requirement
370  leading_type = strip_ws(re.sub("\s*,\s*\(.*?\)\s*$", "", args), qualifier_list)
371  leading_type = first_simple_type_equivalent(leading_type,equiv_classes,equiv_ns,be_typeset)
372  functor_template_types = list([leading_type])
373  args = re.sub(".*?,\s*\(\s*", "", re.sub("\s*\)\s*$", "", args) )
374  for arg in re.findall("[^,]*?\(.*?\)[^,]*?\(.*?\).*?,|[^,]*?<.*?>.*?,|[^,]*?\(.*?\).*?,|[^>\)]*?,", args+","):
375  arg = arg[:-1].strip()
376  if arg != "":
377  if arg == "etc": arg = "..."
378  arg_list = neatsplit('\s',arg)
379  if arg_list[0] in ("class", "struct", "typename"): arg = arg_list[1]
380  arg = first_simple_type_equivalent(strip_ws(arg, qualifier_list),equiv_classes,equiv_ns,be_typeset)
381  functor_template_types.append(arg)
382  else:
383  #This is a backend variable requirement
384  args = first_simple_type_equivalent(strip_ws(args, qualifier_list),equiv_classes,equiv_ns,be_typeset)
385  functor_template_types = list([args+"*"])
386 
387  else:
388  #If function/variable type has qualifier, combine elements in splitline
389  cmd_i = command_index[splitline[0]]
390  if splitline[cmd_i].strip() in qualifier_list:
391  splitline[cmd_i:cmd_i+2] = [" ".join(splitline[cmd_i:cmd_i+2])]
392 
393  functor_template_types = list([strip_ws(splitline[command_index[splitline[0]]], qualifier_list)])
394  functor_template_types[0] = first_simple_type_equivalent(functor_template_types[0],equiv_classes,equiv_ns,be_typeset)
395  if splitline[0].endswith("FUNCTION"):
396  #Get the argument types out of a BE_FUNCTION or BE_CONV_FUNCTION command
397  args = re.sub("\s*BE_(CONV_)?FUNCTION\s*\(.*?,.*?,\s*?\(", "", line)
398  args = re.sub("\([^\(]*?\)\s*\)\s*$", "\)", args)
399  if splitline[0] == "BE_FUNCTION":
400  args = re.sub("\)\s*,[^\)]*?,[^\)]*?\)\s*$", "", args)
401  else:
402  args = re.sub("\)\s*,[^\)]*?\)\s*$", "", args)
403  for arg in re.findall("[^,]*?\(.*?\)[^,]*?\(.*?\).*?,|[^,]*?<.*?>.*?,|[^,]*?\(.*?\).*?,|[^>\)]*?,", args+","):
404  arg = arg[:-1].strip()
405  if arg != "" and not arg.startswith("\"") and not arg.startswith("("):
406  if arg == "etc": arg = "..."
407  arg_list = neatsplit('\s',arg)
408  if arg_list[0] in ("class", "struct", "typename"): arg = arg_list[1]
409  arg = first_simple_type_equivalent(strip_ws(arg, qualifier_list),equiv_classes,equiv_ns,be_typeset)
410  functor_template_types.append(arg)
411  else:
412  #Convert the type to a pointer if this is a backend variable functor rather than a backend function functor
413  functor_template_types[0] += "*"
414 
415  #Iterate over all the candidate types and remove any leading Gambit namespace
416  candidate_types = set(functor_template_types)
417  new_candidate_types = []
418  for candidate_type in candidate_types:
419  new_candidate_types.append(re.sub("^Gambit::", "", candidate_type))
420 
421  #Iterate over all the candidate types and check if they are defined.
422  for candidate_type in new_candidate_types:
423  candidate_type = first_simple_type_equivalent(strip_ws(candidate_type, qualifier_list),equiv_classes,equiv_ns,be_typeset)
424  initial_candidate = candidate_type
425  #Skip to the end if the type is already found.
426  if ("Gambit::"+candidate_type in be_typeset):
427  candidate_type = "Gambit::"+candidate_type
428  elif (candidate_type not in be_typeset):
429  be_typeset.add(candidate_type)
430  # Replace the argument types in the functor_template_types with the fully-qualified versions if required.
431  functor_template_types = [candidate_type if entry == initial_candidate else entry for entry in functor_template_types]
432 
433  ptr_args = ",".join(functor_template_types[1:])
434  arg_list = ",".join([x for x in functor_template_types[1:] if x != "..."])
435  type_pack = functor_template_types[0] + "(*)(" + ptr_args + ")," + functor_template_types[0]
436  if arg_list != "": type_pack += "," + arg_list
437  type_pack_set.add(type_pack)
438 
439 
440 # Harvest the list of rollcall headers to be searched, and the list of type headers to be searched.
def addifbefunctormacro(line, be_typeset, type_pack_set, equiv_classes, equiv_ns, verbose=False)
def first_simple_type_equivalent(candidate_in, equivs, nses, existing)
def neatsplit(regex, string)
def strip_ws(s, qualifiers)
Here is the call graph for this function:
Here is the caller graph for this function:

◆ addiffunctormacro()

def harvesting_tools.addiffunctormacro (   line,
  module,
  all_modules,
  typedict,
  typeheaders,
  intrinsic_types,
  exclude_types,
  equiv_classes,
  equiv_ns,
  verbose = False 
)

Definition at line 268 of file harvesting_tools.py.

References check_for_declaration(), check_for_namespace(), first_simple_type_equivalent(), neatsplit(), Gambit::ColliderBit.print(), readlines_nocomments(), and strip_ws().

Referenced by module_harvester.main().

268 def addiffunctormacro(line,module,all_modules,typedict,typeheaders,intrinsic_types,exclude_types,equiv_classes,equiv_ns,verbose=False):
269 
270  command_index = {"START_FUNCTION":1,
271  "QUICK_FUNCTION":5,
272  "DEPENDENCY":2,
273  "MODEL_CONDITIONAL_DEPENDENCY":2,
274  "INTERPRET_AS_PARENT_DEPENDENCY":2,
275  "INTERPRET_AS_X_DEPENDENCY":3,
276  "START_CONDITIONAL_DEPENDENCY":1,
277  "BE_INI_DEPENDENCY":2,
278  "BE_INI_CONDITIONAL_DEPENDENCY":2}
279 
280  line = re.sub(";", "", line)
281  splitline = neatsplit('\(|\)|,|\s',line)
282 
283  qualifier_list = ["const", "struct"]
284  typeset = typedict["all"]
285 
286  if len(splitline)>1 and splitline[0] in command_index.keys():
287  #This line defines a function and one or more of the arguments defines a candidate type
288  index = command_index[splitline[0]]
289  if splitline[index] in qualifier_list:
290  candidate_types = set([splitline[index]+" "+strip_ws(splitline[index+1], qualifier_list)])
291  else:
292  candidate_types = set([strip_ws(splitline[index], qualifier_list)])
293  if splitline[0]=="QUICK_FUNCTION" and len(splitline)>6:
294  #Get the dep types out of a QUICK_FUNCTION command
295  splitline = re.findall("\(.*?\)",re.sub("QUICK_FUNCTION\(", "", re.sub("\)\)\s*$",")",line) ) )
296  for dep in splitline[1:]:
297  splitdep = neatsplit('\(|\)|,',dep)
298  candidate_types.add(splitdep[1].strip())
299  # Remove excluded types from the set
300  candidate_types.difference_update(exclude_types)
301 
302  #Iterate over all the candidate types and remove any leading Gambit namespace
303  new_candidate_types = []
304  for candidate_type in candidate_types:
305  new_candidate_types.append(re.sub("^Gambit::", "", candidate_type))
306 
307  #Iterate over all the candidate types and check if they are defined.
308  for candidate_type in new_candidate_types:
309  candidate_type = first_simple_type_equivalent(candidate_type,equiv_classes,equiv_ns,typeset)
310  #Skip out now if the type is already found.
311  if (candidate_type in typeset or
312  module+"::"+candidate_type in typeset or
313  "Gambit::"+module+"::"+candidate_type in typeset): continue
314  #If the type is not an intrinsic, check if it is declared in any of the module type headers
315  if (candidate_type not in intrinsic_types):
316  if verbose: print( " {0} located, searching for declaration of {1}...".format(line.strip(),candidate_type) )
317  for header in typeheaders:
318  local_namespace = ""
319  found_declaration = False
320  with io.open(header, encoding='utf-8') as f:
321  for newline in readlines_nocomments(f):
322  splitline = neatsplit('\{|\}|:|;',newline)
323  # Determine the local namespace and look for a class or struct matching the candidate type
324  for i in range(5):
325  if len(splitline)>i:
326  local_namespace = check_for_namespace(splitline[i],local_namespace)
327  if not found_declaration:
328  (found_declaration, candidate_type) = check_for_declaration(splitline[i],module,all_modules,local_namespace,candidate_type)
329  # The loop above misses some of the typedefs, so we need to re-parse the whole line for these.
330  if not found_declaration:
331  (found_declaration, candidate_type) = check_for_declaration(newline,module,all_modules,local_namespace,candidate_type)
332  if found_declaration: break
333  # If the type was declared in this header, and this is a module header, save the type into the list of types for this module.
334  if found_declaration and re.sub(".*?/include/gambit/", "",header).startswith(module):
335  if module not in typedict: typedict[module] = set([])
336  typedict[module].add(candidate_type)
337  break
338  # If the type was not identified with any module, save it as a non-module type.
339  if module not in typedict or candidate_type not in typedict[module]:
340  typedict["non_module"].add(candidate_type)
341  else:
342  # Type is intrinsic, so must be a non-module type.
343  typedict["non_module"].add(candidate_type)
344  # Add the type to the list of all types from everywhere.
345  typeset.add(candidate_type)
346 
347 
348 # Harvest type from a BE_VARIABLE, BE_FUNCTION or BE_CONV_FUNCTION macro call
void print(MixMatrix A)
Helper function to print a matrix.
def first_simple_type_equivalent(candidate_in, equivs, nses, existing)
def addiffunctormacro(line, module, all_modules, typedict, typeheaders, intrinsic_types, exclude_types, equiv_classes, equiv_ns, verbose=False)
def check_for_declaration(input_snippet, module, all_modules, local_namespace, candidate_type)
def neatsplit(regex, string)
def strip_ws(s, qualifiers)
def check_for_namespace(input_snippet, local_namespace)
Here is the call graph for this function:
Here is the caller graph for this function:

◆ addifheader()

def harvesting_tools.addifheader (   line,
  headerset,
  exclude_set,
  verbose = False 
)

Definition at line 202 of file harvesting_tools.py.

References neatsplit(), and Gambit::ColliderBit.print().

Referenced by get_headers().

202 def addifheader(line,headerset,exclude_set,verbose=False):
203  splitline = line.split()
204  if len(splitline)>1 and splitline[0]=="#include":
205  #dig the file name out of the enclosing <> or ""
206  split2 = neatsplit('"|<|>',splitline[1])
207  split3 = neatsplit('/',split2[0])
208  if split2[0] not in exclude_set and split3[-1] not in exclude_set:
209  headerset.add(split2[0])
210  if verbose: print(" Added header '{0}' to set".format(split2[0]))
211 
212 # Harvest module names from rollcall headers
void print(MixMatrix A)
Helper function to print a matrix.
def addifheader(line, headerset, exclude_set, verbose=False)
def neatsplit(regex, string)
Here is the call graph for this function:
Here is the caller graph for this function:

◆ check_for_declaration()

def harvesting_tools.check_for_declaration (   input_snippet,
  module,
  all_modules,
  local_namespace,
  candidate_type 
)

Definition at line 138 of file harvesting_tools.py.

References neatsplit().

Referenced by addiffunctormacro().

138 def check_for_declaration(input_snippet,module,all_modules,local_namespace,candidate_type):
139  splitline = neatsplit('\s|\(|\)|\*|\&|\;',input_snippet)
140  candidate_type = re.sub("^\s*|\s*$", "", candidate_type)
141  candidate_parts = neatsplit('::',re.sub("^.*\s", "", re.sub("<.*>", "", candidate_type)))
142  namespace_parts = neatsplit('::',local_namespace)
143  right_class = False
144  # Work out if we are in the module namespace, and if any sub-namespace matches the candidate type.
145  in_module_and_namespace_matches = False
146  if local_namespace and namespace_parts[0] == module:
147  if candidate_type.startswith(local_namespace): in_module_and_namespace_matches = True
148  if len(namespace_parts) == 1:
149  in_module_and_namespace_matches = True
150  else:
151  addon = 0
152  if candidate_parts[0] == module: addon = 1
153  if candidate_parts[0] == "Gambit": addon = 2
154  if len(candidate_parts) == len(namespace_parts) + addon: in_module_and_namespace_matches = True
155  # Continue only if the input snippet is long enough to contain a declaration and there are no namespace issues
156  if len(splitline) > 1 and (not local_namespace or namespace_parts[0] not in all_modules or in_module_and_namespace_matches):
157  # Look for class/struct declarations
158  if splitline[0] in ["class", "struct"]:
159  allowed_matches = (splitline[1], splitline[1]+"*")
160  if candidate_type in allowed_matches or candidate_parts[0] in allowed_matches:
161  right_class = True
162  elif len(candidate_parts) > 1 and candidate_parts[-1] in allowed_matches:
163  if (candidate_parts[:-1] == namespace_parts[1-len(candidate_parts):]): right_class = True
164  # Look for typedefs
165  if len(splitline)>2 and splitline[0]=="typedef":
166  if (candidate_type in splitline[2:]):
167  right_class = True
168  elif len(candidate_parts) > 1 and candidate_parts[-1] in splitline[2:]:
169  if (candidate_parts[:-1] == namespace_parts[1-len(candidate_parts):]): right_class = True
170  # The class declared at this line matches the candidate class
171  if right_class and local_namespace and local_namespace != "Gambit":
172  main_class = candidate_parts[-1]
173  template_args = re.findall("<.*>\*?", candidate_type)
174  if template_args == []:
175  template_args = ""
176  else:
177  template_args = template_args[0]
178  main_class = re.sub("\*$", "", candidate_parts[-1])
179  qualifiers = re.findall("^.*\s", re.sub("<.*>\*?", "", candidate_type))
180  qualifiers = "" if qualifiers == [] else qualifiers[0]
181  candidate_type = qualifiers + local_namespace + "::" + main_class + template_args
182  return (right_class, candidate_type)
183 
184 # Parse a string to see if it has a namespace declaration
def check_for_declaration(input_snippet, module, all_modules, local_namespace, candidate_type)
def neatsplit(regex, string)
Here is the call graph for this function:
Here is the caller graph for this function:

◆ check_for_namespace()

def harvesting_tools.check_for_namespace (   input_snippet,
  local_namespace 
)

Definition at line 185 of file harvesting_tools.py.

References neatsplit().

Referenced by addiffunctormacro().

185 def check_for_namespace(input_snippet,local_namespace):
186  # Exit if the line just defines a namespace alias
187  if "=" in input_snippet:
188  return local_namespace
189  splitline = neatsplit('\s',input_snippet)
190  if len(splitline)>1:
191  # If the line starts by declaring a namespace, return it appended to the current namespace
192  if splitline[0]=="namespace":
193  if local_namespace.endswith(splitline[1]):
194  return local_namespace
195  elif local_namespace and local_namespace != "Gambit" and splitline[1] not in local_namespace:
196  return local_namespace+"::"+splitline[1]
197  else:
198  return splitline[1]
199  return local_namespace
200 
201 # Harvest header filename from an include statement
def neatsplit(regex, string)
def check_for_namespace(input_snippet, local_namespace)
Here is the call graph for this function:
Here is the caller graph for this function:

◆ comment_remover()

def harvesting_tools.comment_remover (   text)

Definition at line 102 of file harvesting_tools.py.

Referenced by scanner+_harvester.main(), and readlines_nocomments().

102 def comment_remover(text):
103  def replacer(match):
104  s = match.group(0)
105  if s.startswith('/'):
106  return ""
107  else:
108  return s
109  pattern = re.compile(
110  r'//.*?$|/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"',
111  re.DOTALL | re.MULTILINE
112  )
113  return re.sub(pattern, replacer, text[:])
114 
115 # Replacement for f.readlines(), which removes all C/C++ comments from the text before returning a list of all the lines (as f.readlines() does)
def comment_remover(text)
Here is the caller graph for this function:

◆ excluded()

def harvesting_tools.excluded (   string,
  st 
)

Definition at line 125 of file harvesting_tools.py.

125 def excluded(string,st):
126  for x in st:
127  if string.startswith(x): return True
128  return False
129 
130 # Nice sorting function (from http://stackoverflow.com/a/2669120/1447953)
def excluded(string, st)

◆ find_and_harvest_headers()

def harvesting_tools.find_and_harvest_headers (   header_set,
  fullheadlist,
  exclude_set,
  dir_exclude_set,
  verbose = False 
)
Locate 'init_headers' in gambit source tree, then read through them and add any headers that are "include"ed in them to headlist
Args:
header_set - set of file names of headers to parse
fullheadlist - list to which full paths of both init_headers, and any subsequently found headers, should be added.
exclude_set - set of names of headers to ignore if we find them.
dir_exclude_set - set of directory names to skip over during the os.walk

Definition at line 448 of file harvesting_tools.py.

References get_headers(), and Gambit::ColliderBit.print().

Referenced by module_harvester.main().

448 def find_and_harvest_headers(header_set,fullheadlist,exclude_set,dir_exclude_set,verbose=False):
449  """Locate 'init_headers' in gambit source tree, then read through them and add any headers that are "include"ed in them to headlist
450  Args:
451  header_set - set of file names of headers to parse
452  fullheadlist - list to which full paths of both init_headers, and any subsequently found headers, should be added.
453  exclude_set - set of names of headers to ignore if we find them.
454  dir_exclude_set - set of directory names to skip over during the os.walk
455  """
456  full_header_paths=[]
457  # Locate the header in the GAMBIT directory structure...
458  # (we should technically search all the include paths in the make file; could pass these in to this script)
459  # Ignores any headers that cannot be found (assumed to be external libraries, etc.)
460  for root,dirs,files in os.walk(".",topdown=True):
461  # Delete any directories from the traverse list if they are in the exclude list
462  [dirs.remove(d) for d in list(dirs) if d in dir_exclude_set]
463  for name in files:
464  for header in header_set:
465  if os.path.join(root,name).endswith(header):
466  if verbose: print( " Located header '{0}' at path '{1}'".format(name,os.path.join(root,name)) )
467  full_header_paths+=[os.path.join(root,name)]
468 
469  # Add newly found paths to output list
470  fullheadlist+=full_header_paths
471 
472  new_headers=set()
473  for path in full_header_paths:
474  get_headers(path,new_headers,exclude_set,verbose=verbose)
475 
476  # Add headers that we started with to the 'exclude_set' so that we don't search them again.
477  new_exclude_set=set()
478  new_exclude_set.update(exclude_set)
479  new_exclude_set.update(header_set)
480 
481  # Delete elements of 'new_exclude_set' from 'new_headers'
482  new_headers.difference_update(new_exclude_set)
483 
484  # Do this again for all the headers we just found, if we found any
485  if len(new_headers) > 0:
486  if verbose: print( " Harvested the following new headers:" )
487  for header in new_headers:
488  if verbose: print( " "+header )
489  find_and_harvest_headers(new_headers,fullheadlist,new_exclude_set,dir_exclude_set,verbose=verbose)
490 
491 #Search the source tree to determine which modules are present, and write a module_rollcall header if the GAMBIT Core exists.
void print(MixMatrix A)
Helper function to print a matrix.
def get_headers(path, header_set, exclude_set, verbose=False)
def find_and_harvest_headers(header_set, fullheadlist, exclude_set, dir_exclude_set, verbose=False)
Here is the call graph for this function:
Here is the caller graph for this function:

◆ first_simple_type_equivalent()

def harvesting_tools.first_simple_type_equivalent (   candidate_in,
  equivs,
  nses,
  existing 
)

Definition at line 225 of file harvesting_tools.py.

References Gambit::ColliderBit.print().

Referenced by addifbefunctormacro(), and addiffunctormacro().

225 def first_simple_type_equivalent(candidate_in, equivs, nses, existing):
226  if candidate_in in existing: return candidate_in
227  candidate = candidate_in
228  candidate.strip()
229  # Convert the leading BOSSed namespace for the default version to the explicit namespace of the actual version
230  for key in nses:
231  ns_default = key+"_default"+"::"
232  ns_true = key+"_"+nses[key]+"::"
233  if candidate.startswith(ns_default): candidate = ns_true+candidate[len(ns_default):]
234  candidate = re.sub("\s"+ns_default," "+ns_true,candidate)
235  # Exists in the equivalency classes
236  if candidate in equivs:
237  candidate_suffix = ""
238  # Pointer or reference to something that exists in the equivalency classes
239  elif candidate[:-1] in equivs:
240  candidate_suffix = candidate[-1:]
241  candidate = candidate[:-1]
242  # Just not there
243  else:
244  return candidate
245  equivalency_class = equivs[candidate]
246  common_elements = set.intersection(set(equivalency_class), existing)
247  if not common_elements:
248  for index in range(len(equivalency_class)):
249  equivalent = equivalency_class[index]
250  if "," not in equivalent: return equivalent+candidate_suffix
251  print( "Error: all equivalent types found have commas in them! Please typedef one without a comma." )
252  print( "Types are: ", equivalency_class )
253  sys.exit(1)
254  if len(common_elements) != 1:
255  print( "Error: existing types and equivalency class have more than one element in common!" )
256  sys.exit(1)
257  return common_elements.pop()+candidate_suffix
258 
259 # Strips all whitespaces from a string, but re-inserts a single regular space after "const" or "struct".
void print(MixMatrix A)
Helper function to print a matrix.
def first_simple_type_equivalent(candidate_in, equivs, nses, existing)
Here is the call graph for this function:
Here is the caller graph for this function:

◆ get_all_files_with_ext()

def harvesting_tools.get_all_files_with_ext (   verbose,
  starting_dir,
  ext_set,
  kind 
)

Definition at line 535 of file harvesting_tools.py.

References Gambit::ColliderBit.print().

535 def get_all_files_with_ext(verbose,starting_dir,ext_set,kind):
536  results=[]
537  for root,dirs,files in os.walk(starting_dir):
538  for name in files:
539  for ext in ext_set:
540  if name.endswith(ext):
541  if verbose: print( " Located "+kind+" file '{0}' at path '{1}'".format(name,os.path.join(root,name)) )
542  results+=[os.path.join(root, name)]
543  return results
544 
545 #Search a directory for headers that are not excluded.
void print(MixMatrix A)
Helper function to print a matrix.
def get_all_files_with_ext(verbose, starting_dir, ext_set, kind)
Here is the call graph for this function:

◆ get_default_boss_namespaces()

def harvesting_tools.get_default_boss_namespaces ( )

Definition at line 53 of file harvesting_tools.py.

References neatsplit(), and readlines_nocomments().

Referenced by module_harvester.main().

54  result = dict()
55  # Load the default_bossed_version header.
56  with (io.open(default_bossed_versions, encoding='utf-8')) as f:
57  for newline in readlines_nocomments(f):
58  newline = newline.strip()
59  if not newline.startswith("#define"): continue
60  line = neatsplit('\s',newline)
61  if not line[1].startswith("Default_"): continue
62  key = line[1][len("Default_"):]
63  result[key] = line[2]
64  return result
65 
66 # Load type equivalencies yaml file and return a dictionary containing all the equivalency classes.
67 # Just use regex rather than pyYAML, as the latter chokes on :: in scalar entries >:-/
def get_default_boss_namespaces()
def neatsplit(regex, string)
Here is the call graph for this function:
Here is the caller graph for this function:

◆ get_headers()

def harvesting_tools.get_headers (   path,
  header_set,
  exclude_set,
  verbose = False 
)
Parse the file at 'path' and add any headers that are "include"ed therin to the set 'header_set'

Definition at line 441 of file harvesting_tools.py.

References addifheader(), and readlines_nocomments().

Referenced by find_and_harvest_headers().

441 def get_headers(path,header_set,exclude_set,verbose=False):
442  """Parse the file at 'path' and add any headers that are "include"ed therin to the set 'header_set'"""
443  with io.open(path, encoding='utf-8') as f:
444  #print( " Parsing header '{0}' for further includes...".format(path) )
445  for line in readlines_nocomments(f):
446  addifheader(line,header_set,exclude_set,verbose=verbose)
447 
def get_headers(path, header_set, exclude_set, verbose=False)
def addifheader(line, headerset, exclude_set, verbose=False)
Here is the call graph for this function:
Here is the caller graph for this function:

◆ get_type_equivalencies()

def harvesting_tools.get_type_equivalencies (   nses)

Definition at line 68 of file harvesting_tools.py.

References readlines_nocomments().

Referenced by module_harvester.main().

68 def get_type_equivalencies(nses):
69  from collections import defaultdict
70  result = defaultdict(list)
71  # Load the equivalencies yaml file
72  with io.open(equiv_config, encoding='utf-8') as f:
73  for newline in readlines_nocomments(f):
74  newline = newline.strip()
75  if newline == "" or newline.startswith("#"): continue
76  newline = re.sub("^\[\s*|\s*\]", "", newline)
77  equivalency_class = list()
78  for member in re.findall("[^,]*?\(.*?\)[^,]*?\(.*?\).*?,|[^,]*?<.*?>.*?,|[^,]*?\(.*?\).*?,|[^>\)]*?,", newline+","):
79  member = re.sub("\"","",member[:-1].strip())
80  # Convert the leading BOSSed namespace for the default version to the explicit namespace of the actual version
81  for key in nses:
82  ns_default = key+"_default"+"::"
83  ns_true = key+"_"+nses[key]+"::"
84  if member.startswith(ns_default): member = ns_true+member[len(ns_default):]
85  member = re.sub("\s"+ns_default," "+ns_true,member)
86 
87  # If the type is an alias of a native int then add int to the equivalency class
88  if re.match("int[0-9]+_t", member):
89  if ( ctypes.sizeof(ctypes.c_int) == 4 and re.search("32", member) ) or ( ctypes.sizeof(ctypes.c_int) == 2 and re.search("16", member) ) :
90  if 'int' not in equivalency_class:
91  equivalency_class+=['int']
92  if member not in equivalency_class:
93  equivalency_class += [member]
94  for member in equivalency_class: result[member] = equivalency_class
95 
96  # Debug output
97  # print('Type equivalencies:')
98  # print(result)
99  return result
100 
101 # Remove C/C++ comments from 'text' (From http://stackoverflow.com/questions/241327/python-snippet-to-remove-c-and-c-comments)
def get_type_equivalencies(nses)
Here is the call graph for this function:
Here is the caller graph for this function:

◆ make_module_rollcall()

def harvesting_tools.make_module_rollcall (   rollcall_headers,
  verbose 
)

Definition at line 596 of file harvesting_tools.py.

References Gambit::ColliderBit.print().

Referenced by retrieve_rollcall_headers().

596 def make_module_rollcall(rollcall_headers,verbose):
597  towrite = "\
598 // GAMBIT: Global and Modular BSM Inference Tool\n\
599 // *********************************************\n\
600 /// \\file \n\
601 /// \n\
602 /// Compile-time registration of GAMBIT modules. \n\
603 /// \n\
604 /// This file was automatically generated by \n\
605 /// module_harvester.py. Do not modify. \n\
606 /// The content is harvested from your local \n\
607 /// installation. If you want to add a new \n\
608 /// module, just create it and make sure it \n\
609 /// contains a rollcall header, and the \n\
610 /// module_harvester.py script will make \n\
611 /// sure it turns up here. \n\
612 /// \n\
613 /// By 'rollcall header', we mean a file \n\
614 /// myBit/include/gambit/myBit/myBit_rollcall.hpp,\n\
615 /// where myBit is the name of your module. \n\
616 /// \n\
617 /// *********************************************\n\
618 /// \n\
619 /// Authors: \n\
620 /// \n\
621 /// \\author The GAMBIT Collaboration \n\
622 /// \date "+datetime.datetime.now().strftime("%I:%M%p on %B %d, %Y")+"\n\
623 /// \n\
624 /// *********************************************\n\
625  \n\
626 #ifndef __module_rollcall_hpp__ \n\
627 #define __module_rollcall_hpp__ \n\
628  \n\
629 #include \"gambit/Elements/module_macros_incore.hpp\"\n\n"
630 
631  for h in rollcall_headers:
632  towrite+='#include \"{0}\"\n'.format(h)
633  towrite+="\n#endif // defined __module_rollcall_hpp__\n"
634 
635  with open("./Core/include/gambit/Core/module_rollcall.hpp","w") as f:
636  f.write(towrite)
637 
638  if verbose: print( "Found GAMBIT Core. Generated module_rollcall.hpp.\n" )
639 
640 
641 
void print(MixMatrix A)
Helper function to print a matrix.
def make_module_rollcall(rollcall_headers, verbose)
Here is the call graph for this function:
Here is the caller graph for this function:

◆ neatsplit()

def harvesting_tools.neatsplit (   regex,
  string 
)

Definition at line 121 of file harvesting_tools.py.

Referenced by addifbefunctormacro(), addiffunctormacro(), addifheader(), check_for_declaration(), check_for_namespace(), get_default_boss_namespaces(), standalone_facilitator.main(), printer_harvester.main(), scanner+_harvester.main(), module_harvester.main(), and update_module().

121 def neatsplit(regex,string):
122  return [x for x in re.split(regex,string) if x != '']
123 
124 # Check if a string matches the start of any entry in a set
def neatsplit(regex, string)
Here is the caller graph for this function:

◆ readlines_nocomments()

def harvesting_tools.readlines_nocomments (   f)

Definition at line 116 of file harvesting_tools.py.

References comment_remover().

Referenced by addiffunctormacro(), get_default_boss_namespaces(), get_headers(), get_type_equivalencies(), and module_harvester.main().

116 def readlines_nocomments(f):
117  processed_text = comment_remover(f.read())
118  return processed_text.splitlines()
119 
120 # No empties from re.split
def comment_remover(text)
Here is the call graph for this function:
Here is the caller graph for this function:

◆ retrieve_generic_headers()

def harvesting_tools.retrieve_generic_headers (   verbose,
  starting_dir,
  kind,
  excludes,
  exclude_list = [] 
)

Definition at line 546 of file harvesting_tools.py.

References Gambit::ColliderBit.print().

Referenced by collider_harvester.main(), model_harvester.main(), and printer_harvester.main().

546 def retrieve_generic_headers(verbose,starting_dir,kind,excludes,exclude_list=[]):
547  headers=[]
548  for root,dirs,files in os.walk(starting_dir):
549  if root.endswith("shared_includes"): continue
550  for name in files:
551  exclude = False
552  for x in excludes:
553  if name.startswith(x): exclude = True
554  if kind == "BOSSed type" and not name.startswith("loaded_types"): exclude = True
555  if not exclude and (name.endswith(".hpp") or name.endswith(".h") or name.endswith(".hh")):
556  if verbose: print( " Located "+kind+" header '{0}' at path '{1}'".format(name,os.path.join(root,name)) )
557  rel_name = re.sub(".*?/include/", "", os.path.relpath(os.path.join(root,name),starting_dir))
558  headers+=[rel_name]
559  if kind != "BOSSed type": break
560  return headers
561 
562 # Check whether or not two files differ in their contents except for the date line
void print(MixMatrix A)
Helper function to print a matrix.
def retrieve_generic_headers(verbose, starting_dir, kind, excludes, exclude_list=[])
Here is the call graph for this function:
Here is the caller graph for this function:

◆ retrieve_module_type_headers()

def harvesting_tools.retrieve_module_type_headers (   verbose,
  install_dir,
  excludes 
)

Definition at line 515 of file harvesting_tools.py.

References Gambit::ColliderBit.print().

Referenced by module_harvester.main().

515 def retrieve_module_type_headers(verbose,install_dir,excludes):
516  type_headers=[]
517  exclude_dirs=["build",".git","runs","scratch","contrib","Backends"]
518  for root,dirs,files in os.walk(install_dir,topdown=True):
519  [dirs.remove(d) for d in list(dirs) if d in exclude_dirs] # bit confusing, but avoids descending into excluded directories
520  for name in files:
521  if ( (name.lower().endswith("_types.hpp") or
522  name.lower().endswith("_types.h") or
523  name.lower().endswith("_types.hh") ) and name.lower().find("bit") != -1):
524  exclude = False
525  bare_name = re.sub(".*_types\\.[h|hpp|hh]$","",name)
526  for x in excludes:
527  if bare_name.startswith(x): exclude = True
528  if (not exclude):
529  if verbose: print( " Located module type header '{0}' at path '{1}'".format(name,os.path.join(root,name)) )
530  rel_name = re.sub(".*?/include/", "", os.path.relpath(os.path.join(root,name),install_dir))
531  type_headers+=[rel_name]
532  return type_headers
533 
534 #Get all files in a directory tree with one of a given set of extensions
void print(MixMatrix A)
Helper function to print a matrix.
def retrieve_module_type_headers(verbose, install_dir, excludes)
Here is the call graph for this function:
Here is the caller graph for this function:

◆ retrieve_rollcall_headers()

def harvesting_tools.retrieve_rollcall_headers (   verbose,
  install_dir,
  excludes 
)

Definition at line 492 of file harvesting_tools.py.

References make_module_rollcall(), and Gambit::ColliderBit.print().

Referenced by module_harvester.main().

492 def retrieve_rollcall_headers(verbose,install_dir,excludes):
493  rollcall_headers=[]
494  core_exists = False
495  exclude_dirs=["build",".git","runs","scratch","contrib","Backends"]
496  for root,dirs,files in os.walk(install_dir,topdown=True):
497  [dirs.remove(d) for d in list(dirs) if d in exclude_dirs] # bit confusing, but avoids descending into excluded directories
498  if (not core_exists and root == install_dir+"/Core/include/gambit/Core"): core_exists = True
499  for name in files:
500  prefix = re.sub("_rollcall\.h.*", "", name)
501  if ( (name.lower().endswith("_rollcall.hpp") or
502  name.lower().endswith("_rollcall.h") or
503  name.lower().endswith("_rollcall.hh") ) and name.lower().find("bit") != -1 and root.endswith(prefix) ):
504  exclude = False
505  for x in excludes:
506  if name.startswith(x): exclude = True
507  if (not exclude):
508  if verbose: print( " Located module rollcall header '{0}' at path '{1}'".format(name,os.path.join(root,name)) )
509  rel_name = re.sub(".*?/include/", "", os.path.relpath(os.path.join(root,name),install_dir))
510  rollcall_headers+=[rel_name]
511  if core_exists: make_module_rollcall(rollcall_headers,verbose)
512  return rollcall_headers
513 
514 #Search the source tree to determine which modules type headers are present.
void print(MixMatrix A)
Helper function to print a matrix.
def make_module_rollcall(rollcall_headers, verbose)
def retrieve_rollcall_headers(verbose, install_dir, excludes)
Here is the call graph for this function:
Here is the caller graph for this function:

◆ same()

def harvesting_tools.same (   f1,
  f2 
)

Definition at line 563 of file harvesting_tools.py.

Referenced by update_only_if_different().

563 def same(f1,f2):
564  file1 = open(f1,"r") file2 = open(f2,"r")
565  for l1,l2 in zip_longest(file1,file2,fillvalue=''):
566  if l1 != l2:
567  l1nospace = ''.join(l1.split()).lower() #remove spaces and make lowercase
568  #print l1
569  #print l2
570  #print l1nospace
571  if not l1nospace.startswith("#\date") \
572  and not l1nospace.startswith("#\\date") \
573  and not l1nospace.startswith("//\date") \
574  and not l1nospace.startswith("//\\date") \
575  and not l1nospace.startswith("///\date") \
576  and not l1nospace.startswith("///\\date"):
577  #print "Doesn't match!", file1, file2
578  #quit()
579  return False
580  return True
581 
582 # Compare a candidate file to an existing file, replacing only if they differ.
583 
Here is the caller graph for this function:

◆ sorted_nicely()

def harvesting_tools.sorted_nicely (   l)
Sort the given iterable in the way that humans expect.

Definition at line 131 of file harvesting_tools.py.

References int.

131 def sorted_nicely( l ):
132  """ Sort the given iterable in the way that humans expect."""
133  convert = lambda text: int(text) if text.isdigit() else text
134  alphanum_key = lambda key: [ convert(c) for c in re.split('([0-9]+)', key) ]
135  return sorted(l, key = alphanum_key)
136 
137 # Parse a string to see if it has a class, struct or typedef declaration
DS5_MSPCTM DS_INTDOF int

◆ strip_ws()

def harvesting_tools.strip_ws (   s,
  qualifiers 
)

Definition at line 260 of file harvesting_tools.py.

Referenced by addifbefunctormacro(), and addiffunctormacro().

260 def strip_ws(s, qualifiers):
261  for q in qualifiers:
262  s = re.sub(q+"\s*", q+"__TEMP_SPACE__", s)
263  s = re.sub("\s*", "", s)
264  return re.sub("__TEMP_SPACE__", " ", s)
265 
266 
267 # Harvest type from a START_FUNCTION or QUICK_FUNCTION macro call
def strip_ws(s, qualifiers)
Here is the caller graph for this function:

◆ update_module()

def harvesting_tools.update_module (   line,
  module 
)

Definition at line 213 of file harvesting_tools.py.

References neatsplit().

Referenced by module_harvester.main().

213 def update_module(line,module):
214  splitline = neatsplit('\(|\)|,|\s',line)
215  if len(splitline)>2:
216  if splitline[0]=="#define" and splitline[1]=="MODULE":
217  #This line defines a module, return its name instead of bothering to look for a START_FUNCTION
218  return splitline[2]
219  return module
220 
221 # Check for an existing or type equivalent entry. Preferentially return
222 # 1. the existing equivalent entry
223 # 2. the first equivalent entry that does not contain a comma
224 # 3. the original type
def neatsplit(regex, string)
def update_module(line, module)
Here is the call graph for this function:
Here is the caller graph for this function:

◆ update_only_if_different()

def harvesting_tools.update_only_if_different (   existing,
  candidate,
  verbose = True 
)

Definition at line 584 of file harvesting_tools.py.

References Gambit::ColliderBit.print(), and same().

Referenced by standalone_facilitator.main(), printer_harvester.main(), scanner+_harvester.main(), module_harvester.main(), and elements_extras.main().

584 def update_only_if_different(existing, candidate, verbose=True):
585  if not os.path.isfile(existing):
586  shutil.move(candidate,existing)
587  if verbose: print( "\033[1;33m Created "+re.sub("\\.\\/","",existing)+"\033[0m" )
588  elif same(existing, candidate):
589  os.remove(candidate)
590  if verbose: print( "\033[1;33m Existing "+re.sub("\\.\\/","",existing)+" is identical to candidate; leaving it untouched\033[0m" )
591  else:
592  shutil.move(candidate,existing)
593  if verbose: print( "\033[1;33m Updated "+re.sub("\\.\\/","",existing)+"\033[0m" )
594 
595 #Create the module_rollcall header in the Core directory
void print(MixMatrix A)
Helper function to print a matrix.
def update_only_if_different(existing, candidate, verbose=True)
Here is the call graph for this function:
Here is the caller graph for this function:

Variable Documentation

◆ default_bossed_versions

string harvesting_tools.default_bossed_versions = "./Backends/include/gambit/Backends/default_bossed_versions.hpp"

Definition at line 49 of file harvesting_tools.py.

◆ equiv_config

string harvesting_tools.equiv_config = "./config/resolution_type_equivalency_classes.yaml"

Definition at line 50 of file harvesting_tools.py.