Belle II Software  release-06-02-00
Configuration.cc
1 /**************************************************************************
2  * basf2 (Belle II Analysis Software Framework) *
3  * Author: The Belle II Collaboration *
4  * *
5  * See git log for contributors and copyright holders. *
6  * This file is licensed under LGPL-3.0, see LICENSE.md. *
7  **************************************************************************/
8 
9 #include <framework/database/Configuration.h>
10 #include <framework/logging/Logger.h>
11 #include <framework/dataobjects/FileMetaData.h>
12 #include <framework/database/Downloader.h>
13 #include <framework/database/Database.h>
14 #include <framework/utilities/Utils.h>
15 #include <boost/python.hpp>
16 #include <framework/core/PyObjConvUtils.h>
17 #include <framework/core/PyObjROOTUtils.h>
18 #include <boost/algorithm/string.hpp>
19 
20 #include <set>
21 #include <TPython.h>
22 
23 // Current default globaltag when generating events.
24 #define CURRENT_DEFAULT_TAG "release-06-00-07"
25 
26 namespace py = boost::python;
27 
28 namespace {
34  std::vector<std::string> extractStringList(const py::object& obj)
35  {
36  std::vector<std::string> result;
37  Belle2::PyObjConvUtils::iteratePythonObject(obj, [&result](const boost::python::object & item) {
38  py::object str(py::handle<>(PyObject_Str(item.ptr()))); // convert to string
39  py::extract<std::string> extract(str); // and extract
40  result.emplace_back(extract()); // and push back
41  return true;
42  });
43  // done, return
44  return result;
45  }
46 }
47 
48 namespace Belle2::Conditions {
49  boost::python::list& CppOrPyList::ensurePy()
50  {
51  // convert to python list ...
52  if (m_value.index() == 0) {
53  boost::python::list tmp;
54  for (const auto& e : std::get<0>(m_value)) { tmp.append(e); }
55  m_value.emplace<boost::python::list>(std::move(tmp));
56  }
57  return std::get<1>(m_value);
58  }
59 
60  std::vector<std::string>& CppOrPyList::ensureCpp()
61  {
62  // or convert to std::vector ...
63  if (m_value.index() == 1) {
64  std::vector<std::string> tmp = extractStringList(std::get<1>(m_value));
65  m_value.emplace<std::vector<std::string>>(std::move(tmp));
66  }
67  return std::get<0>(m_value);
68  }
69 
70  void CppOrPyList::append(const std::string& element)
71  {
72  std::visit(Utils::VisitOverload{
73  [&element](std::vector<std::string>& list) {list.emplace_back(element);},
74  [&element](boost::python::list & list) {list.append(element);}
75  }, m_value);
76  }
77 
78  void CppOrPyList::prepend(const std::string& element)
79  {
80  std::visit(Utils::VisitOverload{
81  [&element](std::vector<std::string>& list) {list.emplace(list.begin(), element);},
82  [&element](boost::python::list & list) {list.insert(0, element);}
83  }, m_value);
84  }
85 
86  void CppOrPyList::shallowCopy(const boost::python::object& source)
87  {
88  ensurePy().slice(boost::python::_, boost::python::_) = source;
89  }
90 
91  Configuration& Configuration::getInstance()
92  {
93  static Configuration instance;
94  return instance;
95  }
96 
98  {
99  // Backwards compatibility with the existing BELLE2_CONDB_GLOBALTAG
100  // environment variable: If it is set disable replay
101  if (EnvironmentVariables::isSet("BELLE2_CONDB_GLOBALTAG")) {
102  fillFromEnv(m_globalTags, "BELLE2_CONDB_GLOBALTAG", "");
104  }
105  std::string serverList = EnvironmentVariables::get("BELLE2_CONDB_SERVERLIST", "http://belle2db.sdcc.bnl.gov/b2s/rest/");
106  fillFromEnv(m_metadataProviders, "BELLE2_CONDB_METADATA", serverList + " /cvmfs/belle.cern.ch/conditions/database.sqlite");
107  fillFromEnv(m_payloadLocations, "BELLE2_CONDB_PAYLOADS", "/cvmfs/belle.cern.ch/conditions");
108  }
109 
110  void Configuration::reset()
111  {
112  if (m_databaseInitialized) {
113  Database::Instance().reset(true);
114  }
115  *this = Configuration();
116  }
117 
118  std::vector<std::string> Configuration::getDefaultGlobalTags() const
119  {
120  // currently the default globaltag can be overwritten by environment variable
121  // so keep that
122  return EnvironmentVariables::getOrCreateList("BELLE2_CONDB_GLOBALTAG", CURRENT_DEFAULT_TAG);
123  }
124 
125  py::tuple Configuration::getDefaultGlobalTagsPy() const
126  {
127  // same as above but as a python tuple ...
128  py::list list;
129  fillFromEnv(list, "BELLE2_CONDB_GLOBALTAG", CURRENT_DEFAULT_TAG);
130  return py::tuple(list);
131  }
132 
133  void Configuration::setInputMetadata(const std::vector<FileMetaData>& inputMetadata)
134  {
135  ensureEditable();
136  m_inputMetadata = inputMetadata;
137  // make sure the list of globaltags to be used is created but empty
138  m_inputGlobaltags.emplace();
139  // now check for compatibility: make sure all metadata have the same globaltag
140  // setting. Unless we don't have metadata ...
141  if (inputMetadata.empty()) return;
142 
143  std::optional<std::string> inputGlobaltags;
144  for (const auto& metadata : inputMetadata) {
145  if (!inputGlobaltags) {
146  inputGlobaltags = metadata.getDatabaseGlobalTag();
147  } else {
148  if (inputGlobaltags != metadata.getDatabaseGlobalTag()) {
149  B2WARNING("Input files metadata contain incompatible globaltag settings, globaltag replay not possible");
150  // no need to set anything
151  return;
152  }
153  }
154  }
155  // if it's still set and empty we have an empty input list ... warn specifically.
156  if (inputGlobaltags and inputGlobaltags->empty()) {
157  B2WARNING("Input files metadata all have empty globaltag setting, globaltag replay not possible");
158  return;
159  }
160  // set the list of globaltags from the string containing the globaltags
161  boost::split(*m_inputGlobaltags, *inputGlobaltags, boost::is_any_of(","));
162 
163  // HACK: So, we successfully set the input globaltags from the input file,
164  // however we also decided that we want to add new payloads for
165  // boost/invariant mass/beam spot. So if any of the files was created
166  // before the first of October 2019 we assume their globaltag might be
167  // missing these new payloads and we append an extra globaltag containing
168  // just this information with lowest priority. If the files actually had
169  // all payloads these legacy payloads will never be used as they have
170  // lowest priority. Otherwise this should enable running over old files.
171  //
172  // TODO: Once we're sure all files being used contain all payloads remove this.
173  std::optional<std::string> youngest;
174  for (const auto& metadata : inputMetadata) {
175  // Skip release 4 or later files.
176  const std::string& release = metadata.getRelease();
177  if (release.substr(0, 8) == "release-" and
178  release.compare(8, 2, "04", 2) >= 0)
179  continue;
180  // Otherwise, get the date of the youngest file.
181  if (!youngest or * youngest > metadata.getDate()) {
182  youngest = metadata.getDate();
183  }
184  }
185  if (youngest and youngest->compare("2019-12-31") < 0) {
186  B2DEBUG(30, "Enabling legacy IP information globaltag in tag replay");
187  m_inputGlobaltags->emplace_back("Legacy_IP_Information");
188  }
189  // END TODO/HACK
190  }
191 
192  std::vector<std::string> Configuration::getBaseTags() const
193  {
194  // return the list of base tags to be used: Either the default tag
195  // or the list of globaltags from the input files
196  if (not m_inputGlobaltags) return getDefaultGlobalTags();
197  return *m_inputGlobaltags;
198  }
199 
200  std::vector<std::string> Configuration::getFinalListOfTags()
201  {
202  if (m_overrideEnabled) {
203  B2INFO("Global tag override is in effect: input globaltags and default globaltag will be ignored");
204  return m_globalTags.ensureCpp();
205  }
206 
207  auto baseList = getBaseTags();
208  if (m_callback) {
209  // Create a dictionary of keyword arguments for the callback
210  py::dict arguments;
211  // we want a python list of the base tags
212  {
213  py::list baseListPy;
214  for (const auto& tag : baseList) baseListPy.append(tag);
215  arguments["base_tags"] = baseListPy;
216  }
217  // and set the user tags from our list.
218  arguments["user_tags"] = m_globalTags.ensurePy();
219  // and prepare list of metadata. It's None when no replay has been
220  // requested which should mean that we generate events
221  arguments["metadata"] = py::object();
222  // otherwise it's a list of file metadata instances
223  if (m_inputGlobaltags) {
224  py::list metaDataList;
225  for (const auto& m : m_inputMetadata) metaDataList.append(createROOTObjectPyCopy(m));
226  arguments["metadata"] = metaDataList;
227  }
228  // arguments ready, call callback function, python will handle the exceptions
229  py::object retval = (*m_callback)(*py::tuple(), **arguments);
230  // If the return value is not None it should be an iterable
231  // containing the final tag list
232  if (retval != py::object()) {
233  return extractStringList(retval);
234  }
235  // callback returned None so fall back to default
236  }
237  // Default tag replay ... bail if list of globaltags is empty
238  if (baseList.empty()) {
239  if (m_inputGlobaltags) {
240  B2FATAL(R"(No baseline globaltags available.
241  The input files you selected don't have compatible globaltags or an empty
242  globaltag setting. As such globaltag configuration cannot be determined
243  automatically.
244 
245  If you really sure that it is a good idea to process these files together
246  you have to manually override the list of globaltags:
247 
248  >>> basf2.conditions.override_globaltags()
249 )");
250  }else{
251  B2FATAL(R"(No default globaltags available.
252  There is no default globaltag available for processing. This usually means
253  you set the environment variable BELLE2_CONDB_GLOBALTAG to an empty value.
254 
255  As this is unlikely to work for even the most basic functionality this is not
256  directly supported anymore. If you really want to disable any access to the
257  conditions database please configure this explicitly
258 
259  >>> basf2.conditions.metadata_providers = []
260  >>> basf2.conditions.override_globaltags([])
261 )");
262  }
263  }
264  // We have base tags and possibly user tags, so return both
265  std::vector finalList = m_globalTags.ensureCpp();
266  for (const auto& tag : baseList) { finalList.emplace_back(tag); }
267  return finalList;
268  }
269 
270  namespace {
272  boost::python::dict expertSettings(const boost::python::tuple& args, boost::python::dict kwargs)
273  {
274  if (py::len(args) != 1) {
275  // keyword only function: raise typerror on non-keyword arguments
276  PyErr_SetString(PyExc_TypeError, ("expert_settings() takes one positional argument but " +
277  std::to_string(len(args)) + " were given").c_str());
278  py::throw_error_already_set();
279  }
280  Configuration& self = py::extract<Configuration&>(args[0]);
281 
282  py::dict result;
283  // We want to check for a list of names if they exist in the input keyword
284  // arguments. If so, set the new value. In any case, add to output
285  // dictionary. Simplest way: create a variadic lambda with references to the
286  // dictionary and arguments for name, getter and setter.
287  auto checkValue = [&kwargs, &result](const std::string & name, auto setter, auto getter) {
288  using value_type = decltype(getter());
289  if (kwargs.has_key(name)) {
290  value_type value{};
291  py::object object = kwargs[name];
292  try {
293  value = PyObjConvUtils::convertPythonObject(object, value);
294  } catch (std::runtime_error&) {
295  std::stringstream error;
296  error << "Cannot convert argument '" << name << "' to " << PyObjConvUtils::Type<value_type>::name();
297  PyErr_SetString(PyExc_TypeError, error.str().c_str());
298  py::throw_error_already_set();
299  }
300  setter(value);
301  // remove key from kwargs so we can easily check for ones we don't understand later
302  py::delitem(kwargs, py::object(name));
303  }
304  result[name] = PyObjConvUtils::convertToPythonObject(getter());
305  };
306  auto& downloader = Downloader::getDefaultInstance();
307  // That was all the heavy lifting, now just declare all known options :D
308  // I would love to indent this a bit better but astyle has different opinions
309  checkValue("save_payloads",
310  [&self](const std::string & path) { self.setNewPayloadLocation(path);},
311  [&self]() {return self.getNewPayloadLocation();});
312  checkValue("download_cache_location",
313  [&self](const std::string & path) { self.setDownloadCacheDirectory(path);},
314  [&self]() {return self.getDownloadCacheDirectory();});
315  checkValue("download_lock_timeout",
316  [&self](size_t timeout) { self.setDownloadLockTimeout(timeout);},
317  [&self]() { return self.getDownloadLockTimeout();});
318  checkValue("usable_globaltag_states",
319  [&self](const auto & states) { self.setUsableTagStates(states); },
320  [&self]() { return self.getUsableTagStates(); });
321  checkValue("connection_timeout",
322  [&downloader](unsigned int timeout) {downloader.setConnectionTimeout(timeout);},
323  [&downloader]() { return downloader.getConnectionTimeout();});
324  checkValue("stalled_timeout",
325  [&downloader](unsigned int timeout) {downloader.setStalledTimeout(timeout);},
326  [&downloader]() { return downloader.getStalledTimeout();});
327  checkValue("max_retries",
328  [&downloader](unsigned int retries) {downloader.setMaxRetries(retries);},
329  [&downloader]() { return downloader.getMaxRetries();});
330  checkValue("backoff_factor",
331  [&downloader](unsigned int factor) { downloader.setBackoffFactor(factor);},
332  [&downloader]() { return downloader.getBackoffFactor();});
333  // And lastly check if there is something in the kwargs we don't understand ...
334  if (py::len(kwargs) > 0) {
335  std::string message = "Unrecognized keyword arguments: ";
336  auto keys = kwargs.keys();
337  for (int i = 0; i < len(keys); ++i) {
338  if (i > 0) message += ", ";
339  message += py::extract<std::string>(keys[i]);
340  }
341  PyErr_SetString(PyExc_TypeError, message.c_str());
342  py::throw_error_already_set();
343  }
344  return result;
345  }
346  }
347 
348  void Configuration::overrideGlobalTagsPy(const boost::python::list& globalTags)
349  {
350  setGlobalTagsPy(globalTags);
351  m_overrideEnabled = true;
352  }
353 
355  {
356  //don't show c++ signature in python doc to keep it simple
357  py::docstring_options options(true, false, false);
358 
359  void (Configuration::*overrideGTFlag)() = &Configuration::overrideGlobalTags;
360  void (Configuration::*overrideGTList)(const py::list&) = &Configuration::overrideGlobalTagsPy;
361  py::object expert = raw_function(expertSettings);
362  py::class_<Configuration>("ConditionsConfiguration", R"DOC(
363 This class contains all configurations for the conditions database service
364 
365 * which globaltags to use
366 * where to look for payload information
367 * where to find the actual payload files
368 * which temporary testing payloads to use
369 
370 But for most users the only thing they should need to care about is to set the
371 list of additional `globaltags` to use.
372 )DOC")
373  .add_property("override_enabled", &Configuration::overrideEnabled, R"DOC(
374 Indicator whether or not the override of globaltags is enabled. If true then
375 globaltags present in input files will be ignored and only the ones given in
376 `globaltags` will be considered.
377 )DOC")
378  .def("reset", &Configuration::reset, R"DOC(reset()
379 
380 Reset the conditions database configuration to its original state.
381 )DOC")
382  .add_property("default_globaltags", &Configuration::getDefaultGlobalTagsPy, R"DOC(
383 A tuple containing the default globaltags to be used if events are generated without an input file.
384 )DOC")
385  .add_property("globaltags", &Configuration::getGlobalTagsPy, &Configuration::setGlobalTagsPy, R"DOC(
386 List of globaltags to be used. These globaltags will be the ones with highest
387 priority but by default the globaltags used to create the input files or the
388 default globaltag will also be used.
389 
390 The priority of the globaltags in this list is highest first. So the first in
391 the list will be checked first and all other globaltags will only be checked for
392 payloads not found so far.
393 
394 Warning:
395  By default this list contains the globaltags to be used **in addition** to
396  the ones from the input file or the default one if no input file is present.
397  If this is not desirable you need to call `override_globaltags()` to disable
398  any addition or modification of this list.
399 )DOC")
400  .def("append_globaltag", &Configuration::appendGlobalTag, py::args("name"), R"DOC(append_globaltag(name)
401 
402 Append a globaltag to the end of the `globaltags` list. That means it will be
403 the lowest priority of all tags in the list.
404 )DOC")
405  .def("prepend_globaltag", &Configuration::prependGlobalTag, py::args("name"), R"DOC(prepend_globaltag(name)
406 
407 Add a globaltag to the beginning of the `globaltags` list. That means it will be
408 the highest priority of all tags in the list.
409 )DOC")
410  .def("override_globaltags", overrideGTFlag)
411  .def("override_globaltags", overrideGTList, py::args("globaltags"), R"DOC(override_globaltags(list=None)
412 
413 Enable globaltag override. This disables all modification of the globaltag list at the beginning of processing:
414 
415 * the default globaltag or the input file globaltags will be ignored.
416 * any callback set with `set_globaltag_callback` will be ignored.
417 * the list of `globaltags` will be used exactly as it is.
418 
419 Parameters:
420  list (list(str) or None) if given this list will replace the current content of `globaltags`
421 
422 Warning:
423  it's still possible to modify `globaltags` after this call.
424 )DOC")
425  .def("disable_globaltag_replay", &Configuration::disableGlobalTagReplay, R"DOC("disable_globaltag_replay()
426 
427 Disable global tag replay and revert to the old behavior that the default
428 globaltag will be used if no other globaltags are specified.
429 
430 This is a shortcut to just calling
431 
432  >>> conditions.override_globaltags()
433  >>> conditions.globaltags += list(conditions.default_globaltags)
434 
435 )DOC")
436  .def("append_testing_payloads", &Configuration::appendTestingPayloadLocation, py::args("filename"), R"DOC(append_testing_payloads(filename)
437 
438 Append a text file containing local test payloads to the end of the list of
439 `testing_payloads`. This will mean they will have lower priority than payloads
440 in previously defined text files but still higher priority than globaltags.
441 
442 Parameters:
443  filename (str): file containing a local definition of payloads and their
444  intervals of validity for testing
445 
446 Warning:
447  This functionality is strictly for testing purposes. Using local payloads
448  leads to results which cannot be reproduced by anyone else and thus cannot
449  be published.
450 )DOC")
451  .def("prepend_testing_payloads", &Configuration::prependTestingPayloadLocation, py::args("filename"), R"DOC(prepend_testing_payloads(filename)
452 
453 Insert a text file containing local test payloads in the beginning of the list
454 of `testing_payloads`. This will mean they will have higher priority than payloads in
455 previously defined text files as well as higher priority than globaltags.
456 
457 Parameters:
458  filename (str): file containing a local definition of payloads and their
459  intervals of validity for testing
460 
461 Warning:
462  This functionality is strictly for testing purposes. Using local payloads
463  leads to results which cannot be reproduced by anyone else and thus cannot
464  be published.
465 )DOC")
467 List of text files to look for local testing payloads. Each entry should be a
468 text file containing local payloads and their intervals of validity to be used
469 for testing.
470 
471 Payloads found in these files and valid for the current run will have a higher
472 priority than any of the `globaltags`. If a valid payload is present in multiple
473 files the first one in the list will have higher priority.
474 
475 Warning:
476  This functionality is strictly for testing purposes. Using local payloads
477  leads to results which cannot be reproduced by anyone else and thus cannot
478  be published.
479 )DOC")
480  .add_property("metadata_providers", &Configuration::getMetadataProvidersPy, &Configuration::setMetadataProvidersPy, R"DOC(
481 List of metadata providers to use when looking for payload metadata. There are currently two supported providers:
482 
483 1. Central metadata provider to look for payloads in the central conditions database.
484  This provider is used for any entry in this list which starts with ``http(s)://``.
485  The URL should point to the top level of the REST api endpoints on the server
486 
487 2. Local metadata provider to look for payloads in a local SQLite snapshot taken
488  from the central server. This provider will be assumed for any entry in this
489  list not starting with a protocol specifier or if the protocol is given as ``file://``
490 
491 This list should rarely need to be changed. The only exception is for users who
492 want to be able to use the software without internet connection after they
493 downloaded a snapshot of the necessary globaltags with ``b2conditionsdb download``
494 to point to this location.
495 )DOC")
496  .add_property("payload_locations", &Configuration::getPayloadLocationsPy, &Configuration::setPayloadLocationsPy, R"DOC(
497 List of payload locations to search for payloads which have been found by any of
498 the configured `metadata_providers`. This can be a local directory or a
499 ``http(s)://`` url pointing to the payload directory on a server.
500 
501 For remote locations starting with ``http(s)://`` we assume that the layout of
502 the payloads on the server is the same as on the main payload server:
503 The combination of given location and the relative url in the payload metadata
504 field ``payloadUrl`` should point to the correct payload on the server.
505 
506 For local directories, two layouts are supported and will be auto detected:
507 
508 flat
509  All payloads are in the same directory without any substructure with the name
510  ``dbstore_{name}_rev_{revision}.root``
511 hashed
512  All payloads are stored in subdirectories in the form ``AB/{name}_r{revision}.root``
513  where ``A`` and ``B`` are the first two characters of the md5 checksum of the
514  payload file.
515 
516 Example:
517  Given ``payload_locations = ["payload_dir/", "http://server.com/payloads"]``
518  the framework would look for a payload with name ``BeamParameters`` in revision
519  ``45`` (and checksum ``a34ce5...``) in the following places
520 
521 
522  1. ``payload_dir/a3/BeamParameters_r45.root``
523  2. ``payload_dir/dbstore_BeamParameters_rev_45.root``
524  3. ``http://server.com/payloads/dbstore/BeamParameters/dbstore_BeamParameters_rev_45.root``
525  given the usual pattern of the ``payloadUrl`` metadata. But this could be
526  changed on the central servers so mirrors should not depend on this convention
527  but copy the actual structure of the central server.
528 
529 If the payload cannot be found in any of the given locations the framework will
530 always attempt to download it directly from the central server and put it in a
531 local cache directory.
532 )DOC")
533  .def("expert_settings", expert, R"DOC(expert_settings(**kwargs)
534 
535 Set some additional settings for the conditions database.
536 
537 You can supply any combination of keyword-only arguments defined below. The
538 function will return a dictionary containing all current settings.
539 
540  >>> conditions.expert_settings(connection_timeout=5, max_retries=1)
541  {'save_payloads': 'localdb/database.txt',
542  'download_cache_location': '',
543  'download_lock_timeout': 120,
544  'usable_globaltag_states': {'PUBLISHED', 'RUNNING', 'TESTING', 'VALIDATED'},
545  'connection_timeout': 5,
546  'stalled_timeout': 60,
547  'max_retries': 1,
548  'backoff_factor': 5}
549 
550 Warning:
551  Modification of these parameters should not be needed, in rare
552  circumstances this could be used to optimize access for many jobs at once
553  but should only be set by experts.
554 
555 Parameters:
556  save_payloads (str): Where to store new payloads created during processing.
557  This should be a filename to contain the payload information and the payload
558  files will be placed in the same directory as the file.
559  download_cache_location (str): Where to store payloads which have been downloaded
560  from the central server. This could be a user defined directory, otherwise
561  empty string defaults to ``$TMPDIR/basf2-conditions`` where ``$TMPDIR`` is the
562  temporary directories defined in the system. Newly downloaded payloads will
563  be stored in this directory in a hashed structure, see `payload_locations`
564  download_lock_timeout (int): How many seconds to wait for a write lock when
565  concurrently downloading the same payload between different processes.
566  If locking fails the payload will be downloaded to a temporary file
567  separately for each process.
568  usable_globaltag_states (set(str)): Names of globaltag states accepted for
569  processing. This can be changed to make sure that only fully published
570  globaltags are used or to enable running on an open tag. It is not possible
571  to allow usage of 'INVALID' tags, those will always be rejected.
572  connection_timeout (int): timeout in seconds before connection should be
573  aborted. 0 sets the timeout to the default (300s)
574  stalled_timeout (int): timeout in seconds before a download should be
575  aborted if the speed stays below 10 KB/s, 0 disables this timeout
576  max_retries (int): maximum amount of retries if the server responded with
577  an HTTP response of 500 or more. 0 disables retrying
578  backoff_factor (int): backoff factor for retries in seconds. Retries are
579  performed using something similar to binary backoff: For retry :math:`n`
580  and a ``backoff_factor`` :math:`f` we wait for a random time chosen
581  uniformely from the interval :math:`[1, (2^{n} - 1) \times f]` in
582  seconds.
583 )DOC")
584  .def("set_globaltag_callback", &Configuration::setGlobaltagCallbackPy, R"DOC(set_globaltag_callback(function)
585 
586 Set a callback function to be called just before processing.
587 
588 This callback can be used to further customize the globaltags to be used during
589 processing. It will be called after the input files have been opened and checked
590 with three keyword arguments:
591 
592 base_tags
593  The globaltags determined from either the input files or, if no input files
594  are present, the default globaltags
595 
596 user_tags
597  The globaltags provided by the user
598 
599 metadata
600  If there are not input files (e.g. generating events) this argument is None.
601  Otherwise it is a list of all the ``FileMetaData`` instances from all input files.
602  This list can be empty if there is no metadata associated with the input files.
603 
604 From this information the callback function should then compose the final list
605 of globaltags to be used for processing and return this list. If ``None`` is
606 returned the default behavior is applied as if there were no callback function.
607 If anything else is returned the processing is aborted.
608 
609 If no callback function is specified the default behavior is equivalent to ::
610 
611  def callback(base_tags, user_tags, metadata):
612  if not base_tags:
613  basf2.B2FATAL("No baseline globaltags available. Please use override")
614 
615  return user_tags + base_tags
616 
617 If `override_enabled` is ``True`` then the callback function will not be called.
618 
619 Warning:
620  If a callback is set it is responsible to select the correct list of globaltags
621  and also make sure that all files are compatible. No further checks will be
622  done by the framework but any list of globaltags which is returned will be used
623  exactly as it is.
624 
625  If the list of ``base_tags`` is empty that usually means that the input files
626  had different globaltag settings but it is the responsibility of the callback
627  to then verify if the list of globaltags is usable or not.
628 
629  If the callback function determines that no working set of globaltags can be
630  determined then it should abort processing using a FATAL error or an exception
631 )DOC")
632  ;
633 
634  py::scope().attr("conditions") = py::ptr(&Configuration::getInstance());
635  }
636 } // Belle2::Conditions namespace
bool m_overrideEnabled
is the globaltag override enabled?
static Configuration & getInstance()
Get a reference to the instance which will be used when the Database is initialized.
CppOrPyList m_globalTags
the list with all user globaltags
void prependGlobalTag(const std::string &globalTag)
preprend a globaltag
Definition: Configuration.h:79
boost::python::tuple getDefaultGlobalTagsPy() const
Get the tuple of default globaltags as python version.
void appendGlobalTag(const std::string &globalTag)
Append a globaltag.
Definition: Configuration.h:77
Configuration()
Initialize default values.
void ensureEditable() const
Check whether the configuration object can be edited or if the database has been initialized already.
void setGlobaltagCallbackPy(const boost::python::object &obj)
Set a callback function from python which will be called when processing starts and should return the...
std::vector< std::string > getBaseTags() const
Get the base globaltags to be used in addition to user globaltags.
void disableGlobalTagReplay()
Disable global tag replay.
boost::python::list getGlobalTagsPy()
Get the list of user globaltags as python version.
Definition: Configuration.h:87
void setMetadataProvidersPy(const boost::python::list &list)
Set the list of metadata providers in python.
boost::python::list getTestingPayloadLocationsPy()
Get the list of text files containing test payloads in python.
static void fillFromEnv(T &target, const std::string &envName, const std::string &defaultValue)
Fill a target object from a list of environment variables.
std::vector< FileMetaData > m_inputMetadata
the file metadata of all input files if globaltag replay is requested by input module
std::vector< std::string > getFinalListOfTags()
Get the final list of globaltags to be used for processing.
void setPayloadLocationsPy(const boost::python::list &list)
Set the list of payload locations in python.
CppOrPyList m_metadataProviders
the list with all the metadata providers
void setInputMetadata(const std::vector< FileMetaData > &inputMetadata)
To be called by input modules with the list of all input FileMetaData.
boost::python::list getMetadataProvidersPy()
Get the list of metadata providers in python.
void overrideGlobalTags()
Enable globaltag override: If this is called once than overrideEnabled() will return true and getFina...
void setTestingPayloadLocationsPy(const boost::python::list &list)
Set the list of text files containing test payloads in python.
std::optional< std::vector< std::string > > m_inputGlobaltags
the list of globaltags from all the input files to be used in addition to the user globaltags
void prependTestingPayloadLocation(const std::string &filename)
Prepend a local text file with testing payloads to the list.
boost::python::list getPayloadLocationsPy()
Get the list og payload locations in python.
static void exposePythonAPI()
expose this class to python
CppOrPyList m_payloadLocations
the list with all the payload locations
void overrideGlobalTagsPy(const boost::python::list &globalTags)
Enable globaltag override and set the list of user globaltags in one go.
void reset()
Reset to default values.
void setGlobalTagsPy(const boost::python::list &globalTags)
Set the list of globaltags from python.
Definition: Configuration.h:83
bool overrideEnabled() const
Check if override is enabled by previous calls to overrideGlobalTags()
std::optional< boost::python::object > m_callback
the callback function to determine the final final list of globaltags
std::vector< std::string > getDefaultGlobalTags() const
Get the std::vector of default globaltags.
void appendTestingPayloadLocation(const std::string &filename)
Add a local text file with testing payloads.
bool m_databaseInitialized
bool indicating whether the database has been initialized, in which case any changes to the configura...
boost::python::list & ensurePy()
Return the python list version.
void prepend(const std::string &element)
Prepend an element to whatever representation we currently have.
void shallowCopy(const boost::python::object &source)
shallow copy all elements of the source object into the python representation.
void append(const std::string &element)
Append an element to whatever representation we currently have.
std::vector< std::string > & ensureCpp()
Return the C++ vector version.
std::variant< std::vector< std::string >, boost::python::list > m_value
Store either a std::vector or a python list of strings.
Definition: Configuration.h:44
static Downloader & getDefaultInstance()
Return the default instance.
Definition: Downloader.cc:133
static std::string get(const std::string &name, const std::string &fallback="")
Get the value of an environment variable or the given fallback value if the variable is not set.
static bool isSet(const std::string &name)
Check if a value is set in the database.
static Database & Instance()
Instance of a singleton Database.
Definition: Database.cc:41
static std::vector< std::string > getOrCreateList(const std::string &name, const std::string &fallback, const std::string &separators=" \t\n\r")
Get a list of values from an environment variable or the given fallback string if the variable is not...
boost::python::object createROOTObjectPyCopy(const T &instance)
Create a python wrapped copy from a class instance which has a ROOT dictionary.
static void reset(bool keepConfig=false)
Reset the database instance.
Definition: Database.cc:49
boost::python::object convertToPythonObject(const Scalar &value)
------------— From C++ TO Python Converter ---------------------—
Scalar convertPythonObject(const boost::python::object &pyObject, Scalar)
Convert from Python to given type.
bool iteratePythonObject(const boost::python::object &pyObject, Functor function)
Helper function to loop over a python object that implements the iterator concept and call a functor ...
VisitOverload(Ts...) -> VisitOverload< Ts... >
Function for the C++17 std::visit overload pattern to allow simple use of variants.
static std::string name()
type name.