Belle II Software  release-05-01-25
Configuration.cc
1 /**************************************************************************
2  * BASF2 (Belle Analysis Framework 2) *
3  * Copyright(C) 2019 - Belle II Collaboration *
4  * *
5  * Author: The Belle II Collaboration *
6  * Contributors: Martin Ritter *
7  * *
8  * This software is provided "as is" without any warranty. *
9  **************************************************************************/
10 
11 #include <framework/database/Configuration.h>
12 #include <framework/logging/Logger.h>
13 #include <framework/dataobjects/FileMetaData.h>
14 #include <framework/database/Downloader.h>
15 #include <framework/database/Database.h>
16 #include <framework/utilities/Utils.h>
17 #include <boost/python.hpp>
18 #include <framework/core/PyObjConvUtils.h>
19 #include <framework/core/PyObjROOTUtils.h>
20 #include <boost/algorithm/string.hpp>
21 
22 #include <set>
23 #include <TPython.h>
24 
25 // Current default globaltag when generating events.
26 #define CURRENT_DEFAULT_TAG "release-05-01-09"
27 
28 namespace py = boost::python;
29 
30 namespace {
36  std::vector<std::string> extractStringList(const py::object& obj)
37  {
38  std::vector<std::string> result;
39  Belle2::PyObjConvUtils::iteratePythonObject(obj, [&result](const boost::python::object & item) {
40  py::object str(py::handle<>(PyObject_Str(item.ptr()))); // convert to string
41  py::extract<std::string> extract(str); // and extract
42  result.emplace_back(extract()); // and push back
43  return true;
44  });
45  // done, return
46  return result;
47  }
48 }
49 
50 namespace Belle2::Conditions {
51  boost::python::list& CppOrPyList::ensurePy()
52  {
53  // convert to python list ...
54  if (m_value.index() == 0) {
55  boost::python::list tmp;
56  for (const auto& e : std::get<0>(m_value)) { tmp.append(e); }
57  m_value.emplace<boost::python::list>(std::move(tmp));
58  }
59  return std::get<1>(m_value);
60  }
61 
62  std::vector<std::string>& CppOrPyList::ensureCpp()
63  {
64  // or convert to std::vector ...
65  if (m_value.index() == 1) {
66  std::vector<std::string> tmp = extractStringList(std::get<1>(m_value));
67  m_value.emplace<std::vector<std::string>>(std::move(tmp));
68  }
69  return std::get<0>(m_value);
70  }
71 
72  void CppOrPyList::append(const std::string& element)
73  {
74  std::visit(Utils::VisitOverload{
75  [&element](std::vector<std::string>& list) {list.emplace_back(element);},
76  [&element](boost::python::list & list) {list.append(element);}
77  }, m_value);
78  }
79 
80  void CppOrPyList::prepend(const std::string& element)
81  {
82  std::visit(Utils::VisitOverload{
83  [&element](std::vector<std::string>& list) {list.emplace(list.begin(), element);},
84  [&element](boost::python::list & list) {list.insert(0, element);}
85  }, m_value);
86  }
87 
88  void CppOrPyList::shallowCopy(const boost::python::object& source)
89  {
90  ensurePy().slice(boost::python::_, boost::python::_) = source;
91  }
92 
93  Configuration& Configuration::getInstance()
94  {
95  static Configuration instance;
96  return instance;
97  }
98 
100  {
101  // Backwards compatibility with the existing BELLE2_CONDB_GLOBALTAG
102  // environment variable: If it is set disable replay
103  if (EnvironmentVariables::isSet("BELLE2_CONDB_GLOBALTAG")) {
104  fillFromEnv(m_globalTags, "BELLE2_CONDB_GLOBALTAG", "");
106  }
107  std::string serverList = EnvironmentVariables::get("BELLE2_CONDB_SERVERLIST", "http://belle2db.sdcc.bnl.gov/b2s/rest/");
108  fillFromEnv(m_metadataProviders, "BELLE2_CONDB_METADATA", serverList + " /cvmfs/belle.cern.ch/conditions/database.sqlite");
109  fillFromEnv(m_payloadLocations, "BELLE2_CONDB_PAYLOADS", "/cvmfs/belle.cern.ch/conditions");
110  }
111 
112  void Configuration::reset()
113  {
114  if (m_databaseInitialized) {
115  Database::Instance().reset(true);
116  }
117  *this = Configuration();
118  }
119 
120  std::vector<std::string> Configuration::getDefaultGlobalTags() const
121  {
122  // currently the default globaltag can be overwritten by environment variable
123  // so keep that
124  return EnvironmentVariables::getOrCreateList("BELLE2_CONDB_GLOBALTAG", CURRENT_DEFAULT_TAG);
125  }
126 
127  py::tuple Configuration::getDefaultGlobalTagsPy() const
128  {
129  // same as above but as a python tuple ...
130  py::list list;
131  fillFromEnv(list, "BELLE2_CONDB_GLOBALTAG", CURRENT_DEFAULT_TAG);
132  return py::tuple(list);
133  }
134 
135  void Configuration::setInputMetadata(const std::vector<FileMetaData>& inputMetadata)
136  {
137  ensureEditable();
138  m_inputMetadata = inputMetadata;
139  // make sure the list of globaltags to be used is created but empty
140  m_inputGlobaltags.emplace();
141  // now check for compatibility: make sure all metadata have the same globaltag
142  // setting. Unless we don't have metadata ...
143  if (inputMetadata.empty()) return;
144 
145  std::optional<std::string> inputGlobaltags;
146  for (const auto& metadata : inputMetadata) {
147  if (!inputGlobaltags) {
148  inputGlobaltags = metadata.getDatabaseGlobalTag();
149  } else {
150  if (inputGlobaltags != metadata.getDatabaseGlobalTag()) {
151  B2WARNING("Input files metadata contain incompatible globaltag settings, globaltag replay not possible");
152  // no need to set anything
153  return;
154  }
155  }
156  }
157  // if it's still set and empty we have an empty input list ... warn specifically.
158  if (inputGlobaltags and inputGlobaltags->empty()) {
159  B2WARNING("Input files metadata all have empty globaltag setting, globaltag replay not possible");
160  return;
161  }
162  // set the list of globaltags from the string containing the globaltags
163  boost::split(*m_inputGlobaltags, *inputGlobaltags, boost::is_any_of(","));
164 
165  // HACK: So, we successfully set the input globaltags from the input file,
166  // however we also decided that we want to add new payloads for
167  // boost/invariant mass/beam spot. So if any of the files was created
168  // before the first of October 2019 we assume their globaltag might be
169  // missing these new payloads and we append an extra globaltag containing
170  // just this information with lowest priority. If the files actually had
171  // all payloads these legacy payloads will never be used as they have
172  // lowest priority. Otherwise this should enable running over old files.
173  //
174  // TODO: Once we're sure all files being used contain all payloads remove this.
175  std::optional<std::string> youngest;
176  for (const auto& metadata : inputMetadata) {
177  // Skip release 4 or later files.
178  const std::string& release = metadata.getRelease();
179  if (release.substr(0, 8) == "release-" and
180  release.compare(8, 2, "04", 2) >= 0)
181  continue;
182  // Otherwise, get the date of the youngest file.
183  if (!youngest or * youngest > metadata.getDate()) {
184  youngest = metadata.getDate();
185  }
186  }
187  if (youngest and youngest->compare("2019-12-31") < 0) {
188  B2DEBUG(30, "Enabling legacy IP information globaltag in tag replay");
189  m_inputGlobaltags->emplace_back("Legacy_IP_Information");
190  }
191  // END TODO/HACK
192  }
193 
194  std::vector<std::string> Configuration::getBaseTags() const
195  {
196  // return the list of base tags to be used: Either the default tag
197  // or the list of globaltags from the input files
198  if (not m_inputGlobaltags) return getDefaultGlobalTags();
199  return *m_inputGlobaltags;
200  }
201 
202  std::vector<std::string> Configuration::getFinalListOfTags()
203  {
204  if (m_overrideEnabled) {
205  B2INFO("Global tag override is in effect: input globaltags and default globaltag will be ignored");
206  return m_globalTags.ensureCpp();
207  }
208 
209  auto baseList = getBaseTags();
210  if (m_callback) {
211  // Create a dictionary of keyword arguments for the callback
212  py::dict arguments;
213  // we want a python list of the base tags
214  {
215  py::list baseListPy;
216  for (const auto& tag : baseList) baseListPy.append(tag);
217  arguments["base_tags"] = baseListPy;
218  }
219  // and set the user tags from our list.
220  arguments["user_tags"] = m_globalTags.ensurePy();
221  // and prepare list of metadata. It's None when no replay has been
222  // requested which should mean that we generate events
223  arguments["metadata"] = py::object();
224  // otherwise it's a list of file metadata instances
225  if (m_inputGlobaltags) {
226  py::list metaDataList;
227  for (const auto& m : m_inputMetadata) metaDataList.append(createROOTObjectPyCopy(m));
228  arguments["metadata"] = metaDataList;
229  }
230  // arguments ready, call callback function, python will handle the exceptions
231  py::object retval = (*m_callback)(*py::tuple(), **arguments);
232  // If the return value is not None it should be an iterable
233  // containing the final tag list
234  if (retval != py::object()) {
235  return extractStringList(retval);
236  }
237  // callback returned None so fall back to default
238  }
239  // Default tag replay ... bail if list of globaltags is empty
240  if (baseList.empty()) {
241  if (m_inputGlobaltags) {
242  B2FATAL(R"(No baseline globaltags available.
243  The input files you selected don't have compatible globaltags or an empty
244  globaltag setting. As such globaltag configuration cannot be determined
245  automatically.
246 
247  If you really sure that it is a good idea to process these files together
248  you have to manually override the list of globaltags:
249 
250  >>> basf2.conditions.override_globaltags()
251 )");
252  }else{
253  B2FATAL(R"(No default globaltags available.
254  There is no default globaltag available for processing. This usually means
255  you set the environment variable BELLE2_CONDB_GLOBALTAG to an empty value.
256 
257  As this is unlikely to work for even the most basic functionality this is not
258  directly supported anymore. If you really want to disable any access to the
259  conditions database please configure this explicitly
260 
261  >>> basf2.conditions.metadata_providers = []
262  >>> basf2.conditions.override_globaltags([])
263 )");
264  }
265  }
266  // We have base tags and possibly user tags, so return both
267  std::vector finalList = m_globalTags.ensureCpp();
268  for (const auto& tag : baseList) { finalList.emplace_back(tag); }
269  return finalList;
270  }
271 
272  namespace {
274  boost::python::dict expertSettings(const boost::python::tuple& args, boost::python::dict kwargs)
275  {
276  if (py::len(args) != 1) {
277  // keyword only function: raise typerror on non-keyword arguments
278  PyErr_SetString(PyExc_TypeError, ("expert_settings() takes one positional argument but " +
279  std::to_string(len(args)) + " were given").c_str());
280  py::throw_error_already_set();
281  }
282  Configuration& self = py::extract<Configuration&>(args[0]);
283 
284  py::dict result;
285  // We want to check for a list of names if they exist in the input keyword
286  // arguments. If so, set the new value. In any case, add to output
287  // dictionary. Simplest way: create a variadic lambda with references to the
288  // dictionary and arguments for name, getter and setter.
289  auto checkValue = [&kwargs, &result](const std::string & name, auto setter, auto getter) {
290  using value_type = decltype(getter());
291  if (kwargs.has_key(name)) {
292  value_type value{};
293  py::object object = kwargs[name];
294  try {
295  value = PyObjConvUtils::convertPythonObject(object, value);
296  } catch (std::runtime_error&) {
297  std::stringstream error;
298  error << "Cannot convert argument '" << name << "' to " << PyObjConvUtils::Type<value_type>::name();
299  PyErr_SetString(PyExc_TypeError, error.str().c_str());
300  py::throw_error_already_set();
301  }
302  setter(value);
303  // remove key from kwargs so we can easily check for ones we don't understand later
304  py::delitem(kwargs, py::object(name));
305  }
306  result[name] = PyObjConvUtils::convertToPythonObject(getter());
307  };
308  auto& downloader = Downloader::getDefaultInstance();
309  // That was all the heavy lifting, now just declare all known options :D
310  // I would love to indent this a bit better but astyle has different opinions
311  checkValue("save_payloads",
312  [&self](const std::string & path) { self.setNewPayloadLocation(path);},
313  [&self]() {return self.getNewPayloadLocation();});
314  checkValue("download_cache_location",
315  [&self](const std::string & path) { self.setDownloadCacheDirectory(path);},
316  [&self]() {return self.getDownloadCacheDirectory();});
317  checkValue("download_lock_timeout",
318  [&self](size_t timeout) { self.setDownloadLockTimeout(timeout);},
319  [&self]() { return self.getDownloadLockTimeout();});
320  checkValue("usable_globaltag_states",
321  [&self](const auto & states) { self.setUsableTagStates(states); },
322  [&self]() { return self.getUsableTagStates(); });
323  checkValue("connection_timeout",
324  [&downloader](unsigned int timeout) {downloader.setConnectionTimeout(timeout);},
325  [&downloader]() { return downloader.getConnectionTimeout();});
326  checkValue("stalled_timeout",
327  [&downloader](unsigned int timeout) {downloader.setStalledTimeout(timeout);},
328  [&downloader]() { return downloader.getStalledTimeout();});
329  checkValue("max_retries",
330  [&downloader](unsigned int retries) {downloader.setMaxRetries(retries);},
331  [&downloader]() { return downloader.getMaxRetries();});
332  checkValue("backoff_factor",
333  [&downloader](unsigned int factor) { downloader.setBackoffFactor(factor);},
334  [&downloader]() { return downloader.getBackoffFactor();});
335  // And lastly check if there is something in the kwargs we don't understand ...
336  if (py::len(kwargs) > 0) {
337  std::string message = "Unrecognized keyword arguments: ";
338  auto keys = kwargs.keys();
339  for (int i = 0; i < len(keys); ++i) {
340  if (i > 0) message += ", ";
341  message += py::extract<std::string>(keys[i]);
342  }
343  PyErr_SetString(PyExc_TypeError, message.c_str());
344  py::throw_error_already_set();
345  }
346  return result;
347  }
348  }
349 
350  void Configuration::overrideGlobalTagsPy(const boost::python::list& globalTags)
351  {
352  setGlobalTagsPy(globalTags);
353  m_overrideEnabled = true;
354  }
355 
357  {
358  //don't show c++ signature in python doc to keep it simple
359  py::docstring_options options(true, false, false);
360 
361  void (Configuration::*overrideGTFlag)() = &Configuration::overrideGlobalTags;
362  void (Configuration::*overrideGTList)(const py::list&) = &Configuration::overrideGlobalTagsPy;
363  py::object expert = raw_function(expertSettings);
364  py::class_<Configuration>("ConditionsConfiguration", R"DOC(
365 This class contains all configurations for the conditions database service
366 
367 * which globaltags to use
368 * where to look for payload information
369 * where to find the actual payload files
370 * which temporary testing payloads to use
371 
372 But for most users the only thing they should need to care about is to set the
373 list of additional `globaltags` to use.
374 )DOC")
375  .add_property("override_enabled", &Configuration::overrideEnabled, R"DOC(
376 Indicator whether or not the override of globaltags is enabled. If true then
377 globaltags present in input files will be ignored and only the ones given in
378 `globaltags` will be considered.
379 )DOC")
380  .def("reset", &Configuration::reset, R"DOC(reset()
381 
382 Reset the conditions database configuration to its original state.
383 )DOC")
384  .add_property("default_globaltags", &Configuration::getDefaultGlobalTagsPy, R"DOC(
385 A tuple containing the default globaltags to be used if events are generated without an input file.
386 )DOC")
387  .add_property("globaltags", &Configuration::getGlobalTagsPy, &Configuration::setGlobalTagsPy, R"DOC(
388 List of globaltags to be used. These globaltags will be the ones with highest
389 priority but by default the globaltags used to create the input files or the
390 default globaltag will also be used.
391 
392 The priority of the globaltags in this list is highest first. So the first in
393 the list will be checked first and all other globaltags will only be checked for
394 payloads not found so far.
395 
396 Warning:
397  By default this list contains the globaltags to be used **in addition** to
398  the ones from the input file or the default one if no input file is present.
399  If this is not desirable you need to call `override_globaltags()` to disable
400  any addition or modification of this list.
401 )DOC")
402  .def("append_globaltag", &Configuration::appendGlobalTag, py::args("name"), R"DOC(append_globaltag(name)
403 
404 Append a globaltag to the end of the `globaltags` list. That means it will be
405 the lowest priority of all tags in the list.
406 )DOC")
407  .def("prepend_globaltag", &Configuration::prependGlobalTag, py::args("name"), R"DOC(prepend_globaltag(name)
408 
409 Add a globaltag to the beginning of the `globaltags` list. That means it will be
410 the highest priority of all tags in the list.
411 )DOC")
412  .def("override_globaltags", overrideGTFlag)
413  .def("override_globaltags", overrideGTList, py::args("globaltags"), R"DOC(override_globaltags(list=None)
414 
415 Enable globaltag override. This disables all modification of the globaltag list at the beginning of processing:
416 
417 * the default globaltag or the input file globaltags will be ignored.
418 * any callback set with `set_globaltag_callback` will be ignored.
419 * the list of `globaltags` will be used exactly as it is.
420 
421 Parameters:
422  list (list(str) or None) if given this list will replace the current content of `globaltags`
423 
424 Warning:
425  it's still possible to modify `globaltags` after this call.
426 )DOC")
427  .def("disable_globaltag_replay", &Configuration::disableGlobalTagReplay, R"DOC("disable_globaltag_replay()
428 
429 Disable global tag replay and revert to the old behavior that the default
430 globaltag will be used if no other globaltags are specified.
431 
432 This is a shortcut to just calling
433 
434  >>> conditions.override_globaltags()
435  >>> conditions.globaltags += list(conditions.default_globaltags)
436 
437 )DOC")
438  .def("append_testing_payloads", &Configuration::appendTestingPayloadLocation, py::args("filename"), R"DOC(append_testing_payloads(filename)
439 
440 Append a text file containing local test payloads to the end of the list of
441 `testing_payloads`. This will mean they will have lower priority than payloads
442 in previously defined text files but still higher priority than globaltags.
443 
444 Parameters:
445  filename (str): file containing a local definition of payloads and their
446  intervals of validity for testing
447 
448 Warning:
449  This functionality is strictly for testing purposes. Using local payloads
450  leads to results which cannot be reproduced by anyone else and thus cannot
451  be published.
452 )DOC")
453  .def("prepend_testing_payloads", &Configuration::prependTestingPayloadLocation, py::args("filename"), R"DOC(prepend_testing_payloads(filename)
454 
455 Insert a text file containing local test payloads in the beginning of the list
456 of `testing_payloads`. This will mean they will have higher priority than payloads in
457 previously defined text files as well as higher priority than globaltags.
458 
459 Parameters:
460  filename (str): file containing a local definition of payloads and their
461  intervals of validity for testing
462 
463 Warning:
464  This functionality is strictly for testing purposes. Using local payloads
465  leads to results which cannot be reproduced by anyone else and thus cannot
466  be published.
467 )DOC")
469 List of text files to look for local testing payloads. Each entry should be a
470 text file containing local payloads and their intervals of validity to be used
471 for testing.
472 
473 Payloads found in these files and valid for the current run will have a higher
474 priority than any of the `globaltags`. If a valid payload is present in multiple
475 files the first one in the list will have higher priority.
476 
477 Warning:
478  This functionality is strictly for testing purposes. Using local payloads
479  leads to results which cannot be reproduced by anyone else and thus cannot
480  be published.
481 )DOC")
482  .add_property("metadata_providers", &Configuration::getMetadataProvidersPy, &Configuration::setMetadataProvidersPy, R"DOC(
483 List of metadata providers to use when looking for payload metadata. There are currently two supported providers:
484 
485 1. Central metadata provider to look for payloads in the central conditions database.
486  This provider is used for any entry in this list which starts with ``http(s)://``.
487  The URL should point to the top level of the REST api endpoints on the server
488 
489 2. Local metadata provider to look for payloads in a local SQLite snapshot taken
490  from the central server. This provider will be assumed for any entry in this
491  list not starting with a protocol specifier or if the protocol is given as ``file://``
492 
493 This list should rarely need to be changed. The only exception is for users who
494 want to be able to use the software without internet connection after they
495 downloaded a snapshot of the necessary globaltags with ``b2conditionsdb download``
496 to point to this location.
497 )DOC")
498  .add_property("payload_locations", &Configuration::getPayloadLocationsPy, &Configuration::setPayloadLocationsPy, R"DOC(
499 List of payload locations to search for payloads which have been found by any of
500 the configured `metadata_providers`. This can be a local directory or a
501 ``http(s)://`` url pointing to the payload directory on a server.
502 
503 For remote locations starting with ``http(s)://`` we assume that the layout of
504 the payloads on the server is the same as on the main payload server:
505 The combination of given location and the relative url in the payload metadata
506 field ``payloadUrl`` should point to the correct payload on the server.
507 
508 For local directories, two layouts are supported and will be auto detected:
509 
510 flat
511  All payloads are in the same directory without any substructure with the name
512  ``dbstore_{name}_rev_{revision}.root``
513 hashed
514  All payloads are stored in subdirectories in the form ``AB/{name}_r{revision}.root``
515  where ``A`` and ``B`` are the first two characters of the md5 checksum of the
516  payload file.
517 
518 Example:
519  Given ``payload_locations = ["payload_dir/", "http://server.com/payloads"]``
520  the framework would look for a payload with name ``BeamParameters`` in revision
521  ``45`` (and checksum ``a34ce5...``) in the following places
522 
523 
524  1. ``payload_dir/a3/BeamParameters_r45.root``
525  2. ``payload_dir/dbstore_BeamParameters_rev_45.root``
526  3. ``http://server.com/payloads/dbstore/BeamParameters/dbstore_BeamParameters_rev_45.root``
527  given the usual pattern of the ``payloadUrl`` metadata. But this could be
528  changed on the central servers so mirrors should not depend on this convention
529  but copy the actual structure of the central server.
530 
531 If the payload cannot be found in any of the given locations the framework will
532 always attempt to download it directly from the central server and put it in a
533 local cache directory.
534 )DOC")
535  .def("expert_settings", expert, R"DOC(expert_settings(**kwargs)
536 
537 Set some additional settings for the conditions database.
538 
539 You can supply any combination of keyword-only arguments defined below. The
540 function will return a dictionary containing all current settings.
541 
542  >>> conditions.expert_settings(connection_timeout=5, max_retries=1)
543  {'save_payloads': 'localdb/database.txt',
544  'download_cache_location': '',
545  'download_lock_timeout': 120,
546  'usable_globaltag_states': {'PUBLISHED', 'RUNNING', 'TESTING', 'VALIDATED'},
547  'connection_timeout': 5,
548  'stalled_timeout': 60,
549  'max_retries': 1,
550  'backoff_factor': 5}
551 
552 Warning:
553  Modification of these parameters should not be needed, in rare
554  circumstances this could be used to optimize access for many jobs at once
555  but should only be set by experts.
556 
557 Parameters:
558  save_payloads (str): Where to store new payloads created during processing.
559  This should be a filename to contain the payload information and the payload
560  files will be placed in the same directory as the file.
561  download_cache_location (str): Where to store payloads which have been downloaded
562  from the central server. This could be a user defined directory, otherwise
563  empty string defaults to ``$TMPDIR/basf2-conditions`` where ``$TMPDIR`` is the
564  temporary directories defined in the system. Newly downloaded payloads will
565  be stored in this directory in a hashed structure, see `payload_locations`
566  download_lock_timeout (int): How many seconds to wait for a write lock when
567  concurrently downloading the same payload between different processes.
568  If locking fails the payload will be downloaded to a temporary file
569  separately for each process.
570  usable_globaltag_states (set(str)): Names of globaltag states accepted for
571  processing. This can be changed to make sure that only fully published
572  globaltags are used or to enable running on an open tag. It is not possible
573  to allow usage of 'INVALID' tags, those will always be rejected.
574  connection_timeout (int): timeout in seconds before connection should be
575  aborted. 0 sets the timeout to the default (300s)
576  stalled_timeout (int): timeout in seconds before a download should be
577  aborted if the speed stays below 10 KB/s, 0 disables this timeout
578  max_retries (int): maximum amount of retries if the server responded with
579  an HTTP response of 500 or more. 0 disables retrying
580  backoff_factor (int): backoff factor for retries in seconds. Retries are
581  performed using something similar to binary backoff: For retry :math:`n`
582  and a ``backoff_factor`` :math:`f` we wait for a random time chosen
583  uniformely from the interval :math:`[1, (2^{n} - 1) \times f]` in
584  seconds.
585 )DOC")
586  .def("set_globaltag_callback", &Configuration::setGlobaltagCallbackPy, R"DOC(set_globaltag_callback(function)
587 
588 Set a callback function to be called just before processing.
589 
590 This callback can be used to further customize the globaltags to be used during
591 processing. It will be called after the input files have been opened and checked
592 with three keyword arguments:
593 
594 base_tags
595  The globaltags determined from either the input files or, if no input files
596  are present, the default globaltags
597 
598 user_tags
599  The globaltags provided by the user
600 
601 metadata
602  If there are not input files (e.g. generating events) this argument is None.
603  Otherwise it is a list of all the ``FileMetaData`` instances from all input files.
604  This list can be empty if there is no metadata associated with the input files.
605 
606 From this information the callback function should then compose the final list
607 of globaltags to be used for processing and return this list. If ``None`` is
608 returned the default behavior is applied as if there were no callback function.
609 If anything else is returned the processing is aborted.
610 
611 If no callback function is specified the default behavior is equivalent to ::
612 
613  def callback(base_tags, user_tags, metadata):
614  if not base_tags:
615  basf2.B2FATAL("No baseline globaltags available. Please use override")
616 
617  return user_tags + base_tags
618 
619 If `override_enabled` is ``True`` then the callback function will not be called.
620 
621 Warning:
622  If a callback is set it is responsible to select the correct list of globaltags
623  and also make sure that all files are compatible. No further checks will be
624  done by the framework but any list of globaltags which is returned will be used
625  exactly as it is.
626 
627  If the list of ``base_tags`` is empty that usually means that the input files
628  had different globaltag settings but it is the responsibility of the callback
629  to then verify if the list of globaltags is usable or not.
630 
631  If the callback function determines that no working set of globaltags can be
632  determined then it should abort processing using a FATAL error or an exception
633 )DOC")
634  ;
635 
636  py::scope().attr("conditions") = py::ptr(&Configuration::getInstance());
637  }
638 } // Belle2::Conditions namespace
Belle2::Conditions::Configuration::prependGlobalTag
void prependGlobalTag(const std::string &globalTag)
preprend a globaltag
Definition: Configuration.h:89
Belle2::Conditions::Configuration::m_databaseInitialized
bool m_databaseInitialized
bool indicating whether the database has been initialized, in which case any changes to the configura...
Definition: Configuration.h:353
Belle2::Conditions::Configuration::getPayloadLocationsPy
boost::python::list getPayloadLocationsPy()
Get the list og payload locations in python.
Definition: Configuration.h:265
Belle2::EnvironmentVariables::getOrCreateList
static std::vector< std::string > getOrCreateList(const std::string &name, const std::string &fallback, const std::string &separators=" \t\n\r")
Get a list of values from an environment variable or the given fallback string if the variable is not...
Definition: EnvironmentVariables.cc:58
Belle2::Conditions::Configuration::overrideEnabled
bool overrideEnabled() const
Check if override is enabled by previous calls to overrideGlobalTags()
Definition: Configuration.h:152
Belle2::Conditions::CppOrPyList::prepend
void prepend(const std::string &element)
Prepend an element to whatever representation we currently have.
Belle2::PyObjConvUtils::convertToPythonObject
boost::python::object convertToPythonObject(const Scalar &value)
------------— From C++ TO Python Converter ---------------------—
Definition: PyObjConvUtils.h:363
Belle2::Conditions::Configuration::getGlobalTagsPy
boost::python::list getGlobalTagsPy()
Get the list of user globaltags as python version.
Definition: Configuration.h:97
Belle2::Conditions::Configuration::fillFromEnv
static void fillFromEnv(T &target, const std::string &envName, const std::string &defaultValue)
Fill a target object from a list of environment variables.
Definition: Configuration.h:315
Belle2::Conditions::Configuration::overrideGlobalTags
void overrideGlobalTags()
Enable globaltag override: If this is called once than overrideEnabled() will return true and getFina...
Definition: Configuration.h:148
Belle2::Conditions::CppOrPyList::m_value
std::variant< std::vector< std::string >, boost::python::list > m_value
Store either a std::vector or a python list of strings.
Definition: Configuration.h:62
Belle2::Conditions::Configuration::m_overrideEnabled
bool m_overrideEnabled
is the globaltag override enabled?
Definition: Configuration.h:327
Belle2::Utils::VisitOverload
VisitOverload(Ts...) -> VisitOverload< Ts... >
Function for the C++17 std::visit overload pattern to allow simple use of variants.
Belle2::Conditions::Configuration::setGlobaltagCallbackPy
void setGlobaltagCallbackPy(const boost::python::object &obj)
Set a callback function from python which will be called when processing starts and should return the...
Definition: Configuration.h:305
Belle2::Conditions::Configuration::m_metadataProviders
CppOrPyList m_metadataProviders
the list with all the metadata providers
Definition: Configuration.h:338
Belle2::Conditions::CppOrPyList::append
void append(const std::string &element)
Append an element to whatever representation we currently have.
Belle2::createROOTObjectPyCopy
boost::python::object createROOTObjectPyCopy(const T &instance)
Create a python wrapped copy from a class instance which has a ROOT dictionary.
Definition: PyObjROOTUtils.h:36
Belle2::Conditions::Configuration::m_inputMetadata
std::vector< FileMetaData > m_inputMetadata
the file metadata of all input files if globaltag replay is requested by input module
Definition: Configuration.h:332
Belle2::Conditions::CppOrPyList::ensureCpp
std::vector< std::string > & ensureCpp()
Return the C++ vector version.
Belle2::Conditions::Configuration::setMetadataProvidersPy
void setMetadataProvidersPy(const boost::python::list &list)
Set the list of metadata providers in python.
Definition: Configuration.h:237
Belle2::Conditions::Configuration::m_inputGlobaltags
std::optional< std::vector< std::string > > m_inputGlobaltags
the list of globaltags from all the input files to be used in addition to the user globaltags
Definition: Configuration.h:330
Belle2::Conditions::Configuration::exposePythonAPI
static void exposePythonAPI()
expose this class to python
Belle2::Conditions::Configuration::getTestingPayloadLocationsPy
boost::python::list getTestingPayloadLocationsPy()
Get the list of text files containing test payloads in python.
Definition: Configuration.h:213
Belle2::EnvironmentVariables::get
static std::string get(const std::string &name, const std::string &fallback="")
Get the value of an environment variable or the given fallback value if the variable is not set.
Definition: EnvironmentVariables.cc:35
Belle2::Conditions::Configuration::getMetadataProvidersPy
boost::python::list getMetadataProvidersPy()
Get the list of metadata providers in python.
Definition: Configuration.h:241
Belle2::Database::reset
static void reset(bool keepConfig=false)
Reset the database instance.
Definition: Database.cc:62
Belle2::Conditions::Configuration::getBaseTags
std::vector< std::string > getBaseTags() const
Get the base globaltags to be used in addition to user globaltags.
Belle2::Conditions::Configuration::m_callback
std::optional< boost::python::object > m_callback
the callback function to determine the final final list of globaltags
Definition: Configuration.h:350
Belle2::Conditions::Configuration::m_payloadLocations
CppOrPyList m_payloadLocations
the list with all the payload locations
Definition: Configuration.h:340
Belle2::Conditions::Configuration::getInstance
static Configuration & getInstance()
Get a reference to the instance which will be used when the Database is initialized.
Belle2::Conditions::Configuration::getFinalListOfTags
std::vector< std::string > getFinalListOfTags()
Get the final list of globaltags to be used for processing.
Belle2::Conditions::Downloader::getDefaultInstance
static Downloader & getDefaultInstance()
Return the default instance.
Definition: Downloader.cc:143
Belle2::PyObjConvUtils::iteratePythonObject
bool iteratePythonObject(const boost::python::object &pyObject, Functor function)
Helper function to loop over a python object that implements the iterator concept and call a functor ...
Definition: PyObjConvUtils.h:266
Belle2::EnvironmentVariables::isSet
static bool isSet(const std::string &name)
Check if a value is set in the database.
Definition: EnvironmentVariables.cc:29
Belle2::Conditions::Configuration::overrideGlobalTagsPy
void overrideGlobalTagsPy(const boost::python::list &globalTags)
Enable globaltag override and set the list of user globaltags in one go.
Belle2::Conditions::Configuration::getDefaultGlobalTags
std::vector< std::string > getDefaultGlobalTags() const
Get the std::vector of default globaltags.
Belle2::Conditions::Configuration::ensureEditable
void ensureEditable() const
Check whether the configuration object can be edited or if the database has been initialized already.
Definition: Configuration.h:321
Belle2::Conditions::Configuration::appendGlobalTag
void appendGlobalTag(const std::string &globalTag)
Append a globaltag.
Definition: Configuration.h:87
Belle2::PyObjConvUtils::Type::name
static std::string name()
type name.
Definition: PyObjConvUtils.h:156
Belle2::Database::Instance
static Database & Instance()
Instance of a singleton Database.
Definition: Database.cc:54
Belle2::Conditions::Configuration::setInputMetadata
void setInputMetadata(const std::vector< FileMetaData > &inputMetadata)
To be called by input modules with the list of all input FileMetaData.
Belle2::Conditions::Configuration::appendTestingPayloadLocation
void appendTestingPayloadLocation(const std::string &filename)
Add a local text file with testing payloads.
Definition: Configuration.h:203
Belle2::Conditions::CppOrPyList::ensurePy
boost::python::list & ensurePy()
Return the python list version.
Belle2::Conditions::Configuration::Configuration
Configuration()
Initialize default values.
Belle2::Conditions::Configuration::setTestingPayloadLocationsPy
void setTestingPayloadLocationsPy(const boost::python::list &list)
Set the list of text files containing test payloads in python.
Definition: Configuration.h:209
Belle2::Conditions::Configuration::setGlobalTagsPy
void setGlobalTagsPy(const boost::python::list &globalTags)
Set the list of globaltags from python.
Definition: Configuration.h:93
Belle2::Conditions::Configuration::setPayloadLocationsPy
void setPayloadLocationsPy(const boost::python::list &list)
Set the list of payload locations in python.
Definition: Configuration.h:261
Belle2::Conditions::Configuration::prependTestingPayloadLocation
void prependTestingPayloadLocation(const std::string &filename)
Prepend a local text file with testing payloads to the list.
Definition: Configuration.h:205
Belle2::Conditions::Configuration::disableGlobalTagReplay
void disableGlobalTagReplay()
Disable global tag replay.
Definition: Configuration.h:161
Belle2::Conditions::Configuration::m_globalTags
CppOrPyList m_globalTags
the list with all user globaltags
Definition: Configuration.h:334
Belle2::Conditions::Configuration::reset
void reset()
Reset to default values.
Belle2::Conditions::CppOrPyList::shallowCopy
void shallowCopy(const boost::python::object &source)
shallow copy all elements of the source object into the python representation.
Belle2::Conditions::Configuration::getDefaultGlobalTagsPy
boost::python::tuple getDefaultGlobalTagsPy() const
Get the tuple of default globaltags as python version.
Belle2::PyObjConvUtils::convertPythonObject
Scalar convertPythonObject(const boost::python::object &pyObject, Scalar)
Convert from Python to given type.
Definition: PyObjConvUtils.h:510