Belle II Software light-2406-ragdoll
Configuration.cc
1/**************************************************************************
2 * basf2 (Belle II Analysis Software Framework) *
3 * Author: The Belle II Collaboration *
4 * *
5 * See git log for contributors and copyright holders. *
6 * This file is licensed under LGPL-3.0, see LICENSE.md. *
7 **************************************************************************/
8
9#include <framework/database/Configuration.h>
10#include <framework/logging/Logger.h>
11#include <framework/dataobjects/FileMetaData.h>
12#include <framework/database/Downloader.h>
13#include <framework/database/Database.h>
14#include <framework/utilities/Utils.h>
15#include <boost/python.hpp>
16#include <framework/core/PyObjConvUtils.h>
17#include <framework/core/PyObjROOTUtils.h>
18#include <boost/algorithm/string.hpp>
19
20#include <set>
21#include <regex>
22#include <TPython.h>
23
24// Current default globaltag when generating events.
25#define CURRENT_DEFAULT_TAG "main_2024-06-21"
26
27namespace py = boost::python;
28
29namespace {
35 std::vector<std::string> extractStringList(const py::object& obj)
36 {
37 std::vector<std::string> result;
38 Belle2::PyObjConvUtils::iteratePythonObject(obj, [&result](const boost::python::object & item) {
39 py::object str(py::handle<>(PyObject_Str(item.ptr()))); // convert to string
40 py::extract<std::string> extract(str); // and extract
41 result.emplace_back(extract()); // and push back
42 return true;
43 });
44 // done, return
45 return result;
46 }
47}
48
49namespace Belle2::Conditions {
50 boost::python::list& CppOrPyList::ensurePy()
51 {
52 // convert to python list ...
53 if (m_value.index() == 0) {
54 boost::python::list tmp;
55 for (const auto& e : std::get<0>(m_value)) { tmp.append(e); }
56 m_value.emplace<boost::python::list>(std::move(tmp));
57 }
58 return std::get<1>(m_value);
59 }
60
61 std::vector<std::string>& CppOrPyList::ensureCpp()
62 {
63 // or convert to std::vector ...
64 if (m_value.index() == 1) {
65 std::vector<std::string> tmp = extractStringList(std::get<1>(m_value));
66 m_value.emplace<std::vector<std::string>>(std::move(tmp));
67 }
68 return std::get<0>(m_value);
69 }
70
71 void CppOrPyList::append(const std::string& element)
72 {
73 std::visit(Utils::VisitOverload{
74 [&element](std::vector<std::string>& list) {list.emplace_back(element);},
75 [&element](boost::python::list & list) {list.append(element);}
76 }, m_value);
77 }
78
79 void CppOrPyList::prepend(const std::string& element)
80 {
81 std::visit(Utils::VisitOverload{
82 [&element](std::vector<std::string>& list) {list.emplace(list.begin(), element);},
83 [&element](boost::python::list & list) {list.insert(0, element);}
84 }, m_value);
85 }
86
87 void CppOrPyList::shallowCopy(const boost::python::object& source)
88 {
89 ensurePy().slice(boost::python::_, boost::python::_) = source;
90 }
91
92 Configuration& Configuration::getInstance()
93 {
94 static Configuration instance;
95 return instance;
96 }
97
99 {
100 // Backwards compatibility with the existing BELLE2_CONDB_GLOBALTAG
101 // environment variable: If it is set disable replay
102 if (EnvironmentVariables::isSet("BELLE2_CONDB_GLOBALTAG")) {
103 fillFromEnv(m_globalTags, "BELLE2_CONDB_GLOBALTAG", "");
105 }
106 std::string serverList = EnvironmentVariables::get("BELLE2_CONDB_SERVERLIST", m_defaultMetadataProviderUrl);
107 fillFromEnv(m_metadataProviders, "BELLE2_CONDB_METADATA", serverList + " /cvmfs/belle.cern.ch/conditions/database.sqlite");
108 fillFromEnv(m_payloadLocations, "BELLE2_CONDB_PAYLOADS", "/cvmfs/belle.cern.ch/conditions");
109 }
110
112 {
115 }
116 *this = Configuration();
117 }
118
119 std::vector<std::string> Configuration::getDefaultGlobalTags() const
120 {
121 // currently the default globaltag can be overwritten by environment variable
122 // so keep that
123 return EnvironmentVariables::getOrCreateList("BELLE2_CONDB_GLOBALTAG", CURRENT_DEFAULT_TAG);
124 }
125
127 {
128 // same as above but as a python tuple ...
129 py::list list;
130 fillFromEnv(list, "BELLE2_CONDB_GLOBALTAG", CURRENT_DEFAULT_TAG);
131 return py::tuple(list);
132 }
133
134 void Configuration::setInputMetadata(const std::vector<FileMetaData>& inputMetadata)
135 {
137 m_inputMetadata = inputMetadata;
138 // make sure the list of globaltags to be used is created but empty
139 m_inputGlobaltags.emplace();
140 // now check for compatibility: make sure all metadata have the same globaltag
141 // setting. Unless we don't have metadata ...
142 if (inputMetadata.empty()) return;
143
144 std::optional<std::string> inputGlobaltags;
145 for (const auto& metadata : inputMetadata) {
146 if (!inputGlobaltags) {
147 inputGlobaltags = metadata.getDatabaseGlobalTag();
148 } else {
149 if (inputGlobaltags != metadata.getDatabaseGlobalTag()) {
150 B2WARNING("Input files metadata contain incompatible globaltag settings, globaltag replay not possible");
151 // no need to set anything
152 return;
153 }
154 }
155 }
156 // if it's still set and empty we have an empty input list ... warn specifically.
157 if (inputGlobaltags and inputGlobaltags->empty()) {
158 B2WARNING("Input files metadata all have empty globaltag setting, globaltag replay not possible");
159 return;
160 }
161 // set the list of globaltags from the string containing the globaltags
162 boost::split(*m_inputGlobaltags, *inputGlobaltags, boost::is_any_of(","));
163
164 // HACK: So, we successfully set the input globaltags from the input file,
165 // however we also decided that we want to add new payloads for
166 // boost, invariant mass, beam spot, collision axis in CMS.
167 // So if the release is older than when these features were introduced
168 // or if files were produced before specific date, extra GTs are appended.
169 // The appended GTs contain only the possible missing info and are added
170 // with lowest priority.
171 // If the files actually had all payloads these legacy payloads will never
172 // be used as they have lowest priority.
173 // Otherwise this should enable running over old files.
174 //
175 // TODO: Once we're sure all files being used contain all payloads remove this.
176
177 std::optional<std::string> relMin, dateMin;
178
179 for (const auto& metadata : inputMetadata) {
180 // get oldest release
181 std::string rel = metadata.getRelease().substr(0, 10);
182 if (std::regex_match(rel, std::regex("release-[0-9][0-9]"))) {
183 if (!relMin) relMin = rel;
184 relMin = min(*relMin, rel);
185 }
186
187 // get oldest production date
188 std::string date = metadata.getDate().substr(0, 10);
189 if (std::regex_match(date, std::regex("[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]"))) {
190 if (!dateMin) dateMin = date;
191 dateMin = min(*dateMin, date);
192 }
193 }
194
195 // add IP GT if rel older than rel04 or for old files
196 if ((relMin && relMin < "release-04") ||
197 (!relMin && (!dateMin || dateMin < "2019-12-31"))) {
198 B2DEBUG(30, "Enabling legacy IP information globaltag in tag replay");
199 m_inputGlobaltags->emplace_back("Legacy_IP_Information");
200 }
201
202 // add CollisionAxisCMS GT if rel older than rel08 or for old files
203 if ((relMin && relMin < "release-08") ||
204 (!relMin && (!dateMin || dateMin < "2023-08-31"))) {
205 B2DEBUG(30, "Enabling legacy CollsionAxisCMS globaltag in tag replay");
206 m_inputGlobaltags->emplace_back("Legacy_CollisionAxisCMS");
207 }
208 // END TODO/HACK
209 }
210
211 std::vector<std::string> Configuration::getBaseTags() const
212 {
213 // return the list of base tags to be used: Either the default tag
214 // or the list of globaltags from the input files
215 if (not m_inputGlobaltags) return getDefaultGlobalTags();
216 return *m_inputGlobaltags;
217 }
218
219 std::vector<std::string> Configuration::getFinalListOfTags()
220 {
221 if (m_overrideEnabled) {
222 B2INFO("Global tag override is in effect: input globaltags and default globaltag will be ignored");
223 return m_globalTags.ensureCpp();
224 }
225
226 auto baseList = getBaseTags();
227 if (m_callback) {
228 // Create a dictionary of keyword arguments for the callback
229 py::dict arguments;
230 // we want a python list of the base tags
231 {
232 py::list baseListPy;
233 for (const auto& tag : baseList) baseListPy.append(tag);
234 arguments["base_tags"] = baseListPy;
235 }
236 // and set the user tags from our list.
237 arguments["user_tags"] = m_globalTags.ensurePy();
238 // and prepare list of metadata. It's None when no replay has been
239 // requested which should mean that we generate events
240 arguments["metadata"] = py::object();
241 // otherwise it's a list of file metadata instances
242 if (m_inputGlobaltags) {
243 py::list metaDataList;
244 for (const auto& m : m_inputMetadata) metaDataList.append(createROOTObjectPyCopy(m));
245 arguments["metadata"] = metaDataList;
246 }
247 // arguments ready, call callback function, python will handle the exceptions
248 py::object retval = (*m_callback)(*py::tuple(), **arguments);
249 // If the return value is not None it should be an iterable
250 // containing the final tag list
251 if (retval != py::object()) {
252 return extractStringList(retval);
253 }
254 // callback returned None so fall back to default
255 }
256 // Default tag replay ... bail if list of globaltags is empty
257 if (baseList.empty()) {
258 if (m_inputGlobaltags) {
259 B2FATAL(R"(No baseline globaltags available.
260 The input files you selected don't have compatible globaltags or an empty
261 globaltag setting. As such globaltag configuration cannot be determined
262 automatically.
263
264 If you really sure that it is a good idea to process these files together
265 you have to manually override the list of globaltags:
266
267 >>> basf2.conditions.override_globaltags()
268)");
269 }else{
270 B2FATAL(R"(No default globaltags available.
271 There is no default globaltag available for processing. This usually means
272 you set the environment variable BELLE2_CONDB_GLOBALTAG to an empty value.
273
274 As this is unlikely to work for even the most basic functionality this is not
275 directly supported anymore. If you really want to disable any access to the
276 conditions database please configure this explicitly
277
278 >>> basf2.conditions.metadata_providers = []
279 >>> basf2.conditions.override_globaltags([])
280)");
281 }
282 }
283 // We have base tags and possibly user tags, so return both
284 std::vector finalList = m_globalTags.ensureCpp();
285 for (const auto& tag : baseList) { finalList.emplace_back(tag); }
286 return finalList;
287 }
288
289 namespace {
291 boost::python::dict expertSettings(const boost::python::tuple& args, boost::python::dict kwargs)
292 {
293 if (py::len(args) != 1) {
294 // keyword only function: raise typerror on non-keyword arguments
295 PyErr_SetString(PyExc_TypeError, ("expert_settings() takes one positional argument but " +
296 std::to_string(len(args)) + " were given").c_str());
297 py::throw_error_already_set();
298 }
299 Configuration& self = py::extract<Configuration&>(args[0]);
300
301 py::dict result;
302 // We want to check for a list of names if they exist in the input keyword
303 // arguments. If so, set the new value. In any case, add to output
304 // dictionary. Simplest way: create a variadic lambda with references to the
305 // dictionary and arguments for name, getter and setter.
306 auto checkValue = [&kwargs, &result](const std::string & name, auto setter, auto getter) {
307 using value_type = decltype(getter());
308 if (kwargs.has_key(name)) {
309 value_type value{};
310 py::object object = kwargs[name];
311 try {
312 value = PyObjConvUtils::convertPythonObject(object, value);
313 } catch (std::runtime_error&) {
314 std::stringstream error;
315 error << "Cannot convert argument '" << name << "' to " << PyObjConvUtils::Type<value_type>::name();
316 PyErr_SetString(PyExc_TypeError, error.str().c_str());
317 py::throw_error_already_set();
318 }
319 setter(value);
320 // remove key from kwargs so we can easily check for ones we don't understand later
321 py::delitem(kwargs, py::object(name));
322 }
323 result[name] = PyObjConvUtils::convertToPythonObject(getter());
324 };
325 auto& downloader = Downloader::getDefaultInstance();
326 // That was all the heavy lifting, now just declare all known options :D
327 // I would love to indent this a bit better but astyle has different opinions
328 checkValue("save_payloads",
329 [&self](const std::string & path) { self.setNewPayloadLocation(path);},
330 [&self]() {return self.getNewPayloadLocation();});
331 checkValue("download_cache_location",
332 [&self](const std::string & path) { self.setDownloadCacheDirectory(path);},
333 [&self]() {return self.getDownloadCacheDirectory();});
334 checkValue("download_lock_timeout",
335 [&self](size_t timeout) { self.setDownloadLockTimeout(timeout);},
336 [&self]() { return self.getDownloadLockTimeout();});
337 checkValue("usable_globaltag_states",
338 [&self](const auto & states) { self.setUsableTagStates(states); },
339 [&self]() { return self.getUsableTagStates(); });
340 checkValue("connection_timeout",
341 [&downloader](unsigned int timeout) {downloader.setConnectionTimeout(timeout);},
342 [&downloader]() { return downloader.getConnectionTimeout();});
343 checkValue("stalled_timeout",
344 [&downloader](unsigned int timeout) {downloader.setStalledTimeout(timeout);},
345 [&downloader]() { return downloader.getStalledTimeout();});
346 checkValue("max_retries",
347 [&downloader](unsigned int retries) {downloader.setMaxRetries(retries);},
348 [&downloader]() { return downloader.getMaxRetries();});
349 checkValue("backoff_factor",
350 [&downloader](unsigned int factor) { downloader.setBackoffFactor(factor);},
351 [&downloader]() { return downloader.getBackoffFactor();});
352 // And lastly check if there is something in the kwargs we don't understand ...
353 if (py::len(kwargs) > 0) {
354 std::string message = "Unrecognized keyword arguments: ";
355 auto keys = kwargs.keys();
356 for (int i = 0; i < len(keys); ++i) {
357 if (i > 0) message += ", ";
358 message += py::extract<std::string>(keys[i]);
359 }
360 PyErr_SetString(PyExc_TypeError, message.c_str());
361 py::throw_error_already_set();
362 }
363 return result;
364 }
365 }
366
367 void Configuration::overrideGlobalTagsPy(const boost::python::list& globalTags)
368 {
369 setGlobalTagsPy(globalTags);
370 m_overrideEnabled = true;
371 }
372
374 {
375 //don't show c++ signature in python doc to keep it simple
376 py::docstring_options options(true, false, false);
377
378 void (Configuration::*overrideGTFlag)() = &Configuration::overrideGlobalTags;
379 void (Configuration::*overrideGTList)(const py::list&) = &Configuration::overrideGlobalTagsPy;
380 py::object expert = raw_function(expertSettings);
381 py::class_<Configuration>("ConditionsConfiguration", R"DOC(
382This class contains all configurations for the conditions database service
383
384* which globaltags to use
385* where to look for payload information
386* where to find the actual payload files
387* which temporary testing payloads to use
388
389But for most users the only thing they should need to care about is to set the
390list of additional `globaltags` to use.
391)DOC")
392 .add_property("override_enabled", &Configuration::overrideEnabled, R"DOC(
393Indicator whether or not the override of globaltags is enabled. If true then
394globaltags present in input files will be ignored and only the ones given in
395`globaltags` will be considered.
396)DOC")
397 .def("reset", &Configuration::reset, R"DOC(reset()
398
399Reset the conditions database configuration to its original state.
400)DOC")
401 .add_property("default_globaltags", &Configuration::getDefaultGlobalTagsPy, R"DOC(
402A tuple containing the default globaltags to be used if events are generated without an input file.
403)DOC")
404 .add_property("globaltags", &Configuration::getGlobalTagsPy, &Configuration::setGlobalTagsPy, R"DOC(
405List of globaltags to be used. These globaltags will be the ones with highest
406priority but by default the globaltags used to create the input files or the
407default globaltag will also be used.
408
409The priority of the globaltags in this list is highest first. So the first in
410the list will be checked first and all other globaltags will only be checked for
411payloads not found so far.
412
413Warning:
414 By default this list contains the globaltags to be used **in addition** to
415 the ones from the input file or the default one if no input file is present.
416 If this is not desirable you need to call `override_globaltags()` to disable
417 any addition or modification of this list.
418)DOC")
419 .def("append_globaltag", &Configuration::appendGlobalTag, py::args("name"), R"DOC(append_globaltag(name)
420
421Append a globaltag to the end of the `globaltags` list. That means it will be
422the lowest priority of all tags in the list.
423)DOC")
424 .def("prepend_globaltag", &Configuration::prependGlobalTag, py::args("name"), R"DOC(prepend_globaltag(name)
425
426Add a globaltag to the beginning of the `globaltags` list. That means it will be
427the highest priority of all tags in the list.
428)DOC")
429 .def("override_globaltags", overrideGTFlag)
430 .def("override_globaltags", overrideGTList, py::args("globaltags"), R"DOC(override_globaltags(list=None)
431
432Enable globaltag override. This disables all modification of the globaltag list at the beginning of processing:
433
434* the default globaltag or the input file globaltags will be ignored.
435* any callback set with `set_globaltag_callback` will be ignored.
436* the list of `globaltags` will be used exactly as it is.
437
438Parameters:
439 list (list(str) or None) if given this list will replace the current content of `globaltags`
440
441Warning:
442 it's still possible to modify `globaltags` after this call.
443)DOC")
444 .def("disable_globaltag_replay", &Configuration::disableGlobalTagReplay, R"DOC("disable_globaltag_replay()
445
446Disable global tag replay and revert to the old behavior that the default
447globaltag will be used if no other globaltags are specified.
448
449This is a shortcut to just calling
450
451 >>> conditions.override_globaltags()
452 >>> conditions.globaltags += list(conditions.default_globaltags)
453
454)DOC")
455 .def("append_testing_payloads", &Configuration::appendTestingPayloadLocation, py::args("filename"), R"DOC(append_testing_payloads(filename)
456
457Append a text file containing local test payloads to the end of the list of
458`testing_payloads`. This will mean they will have lower priority than payloads
459in previously defined text files but still higher priority than globaltags.
460
461Parameters:
462 filename (str): file containing a local definition of payloads and their
463 intervals of validity for testing
464
465Warning:
466 This functionality is strictly for testing purposes. Using local payloads
467 leads to results which cannot be reproduced by anyone else and thus cannot
468 be published.
469)DOC")
470 .def("prepend_testing_payloads", &Configuration::prependTestingPayloadLocation, py::args("filename"), R"DOC(prepend_testing_payloads(filename)
471
472Insert a text file containing local test payloads in the beginning of the list
473of `testing_payloads`. This will mean they will have higher priority than payloads in
474previously defined text files as well as higher priority than globaltags.
475
476Parameters:
477 filename (str): file containing a local definition of payloads and their
478 intervals of validity for testing
479
480Warning:
481 This functionality is strictly for testing purposes. Using local payloads
482 leads to results which cannot be reproduced by anyone else and thus cannot
483 be published.
484)DOC")
486List of text files to look for local testing payloads. Each entry should be a
487text file containing local payloads and their intervals of validity to be used
488for testing.
489
490Payloads found in these files and valid for the current run will have a higher
491priority than any of the `globaltags`. If a valid payload is present in multiple
492files the first one in the list will have higher priority.
493
494Warning:
495 This functionality is strictly for testing purposes. Using local payloads
496 leads to results which cannot be reproduced by anyone else and thus cannot
497 be published.
498)DOC")
499 .add_property("metadata_providers", &Configuration::getMetadataProvidersPy, &Configuration::setMetadataProvidersPy, R"DOC(
500List of metadata providers to use when looking for payload metadata. There are currently two supported providers:
501
5021. Central metadata provider to look for payloads in the central conditions database.
503 This provider is used for any entry in this list which starts with ``http(s)://``.
504 The URL should point to the top level of the REST api endpoints on the server
505
5062. Local metadata provider to look for payloads in a local SQLite snapshot taken
507 from the central server. This provider will be assumed for any entry in this
508 list not starting with a protocol specifier or if the protocol is given as ``file://``
509
510This list should rarely need to be changed. The only exception is for users who
511want to be able to use the software without internet connection after they
512downloaded a snapshot of the necessary globaltags with ``b2conditionsdb download``
513to point to this location.
514)DOC")
515 .add_property("default_metadata_provider_url", &Configuration::getDefaultMetadataProviderUrl, R"DOC(
516URL of the default central metadata provider to look for payloads in the
517conditions database.
518)DOC")
519 .add_property("payload_locations", &Configuration::getPayloadLocationsPy, &Configuration::setPayloadLocationsPy, R"DOC(
520List of payload locations to search for payloads which have been found by any of
521the configured `metadata_providers`. This can be a local directory or a
522``http(s)://`` url pointing to the payload directory on a server.
523
524For remote locations starting with ``http(s)://`` we assume that the layout of
525the payloads on the server is the same as on the main payload server:
526The combination of given location and the relative url in the payload metadata
527field ``payloadUrl`` should point to the correct payload on the server.
528
529For local directories, two layouts are supported and will be auto detected:
530
531flat
532 All payloads are in the same directory without any substructure with the name
533 ``dbstore_{name}_rev_{revision}.root``
534hashed
535 All payloads are stored in subdirectories in the form ``AB/{name}_r{revision}.root``
536 where ``A`` and ``B`` are the first two characters of the md5 checksum of the
537 payload file.
538
539Example:
540 Given ``payload_locations = ["payload_dir/", "http://server.com/payloads"]``
541 the framework would look for a payload with name ``BeamParameters`` in revision
542 ``45`` (and checksum ``a34ce5...``) in the following places
543
544
545 1. ``payload_dir/a3/BeamParameters_r45.root``
546 2. ``payload_dir/dbstore_BeamParameters_rev_45.root``
547 3. ``http://server.com/payloads/dbstore/BeamParameters/dbstore_BeamParameters_rev_45.root``
548 given the usual pattern of the ``payloadUrl`` metadata. But this could be
549 changed on the central servers so mirrors should not depend on this convention
550 but copy the actual structure of the central server.
551
552If the payload cannot be found in any of the given locations the framework will
553always attempt to download it directly from the central server and put it in a
554local cache directory.
555)DOC")
556 .def("expert_settings", expert, R"DOC(expert_settings(**kwargs)
557
558Set some additional settings for the conditions database.
559
560You can supply any combination of keyword-only arguments defined below. The
561function will return a dictionary containing all current settings.
562
563 >>> conditions.expert_settings(connection_timeout=5, max_retries=1)
564 {'save_payloads': 'localdb/database.txt',
565 'download_cache_location': '',
566 'download_lock_timeout': 120,
567 'usable_globaltag_states': {'PUBLISHED', 'RUNNING', 'TESTING', 'VALIDATED'},
568 'connection_timeout': 5,
569 'stalled_timeout': 60,
570 'max_retries': 1,
571 'backoff_factor': 5}
572
573Warning:
574 Modification of these parameters should not be needed, in rare
575 circumstances this could be used to optimize access for many jobs at once
576 but should only be set by experts.
577
578Parameters:
579 save_payloads (str): Where to store new payloads created during processing.
580 This should be a filename to contain the payload information and the payload
581 files will be placed in the same directory as the file.
582 download_cache_location (str): Where to store payloads which have been downloaded
583 from the central server. This could be a user defined directory, otherwise
584 empty string defaults to ``$TMPDIR/basf2-conditions`` where ``$TMPDIR`` is the
585 temporary directories defined in the system. Newly downloaded payloads will
586 be stored in this directory in a hashed structure, see `payload_locations`
587 download_lock_timeout (int): How many seconds to wait for a write lock when
588 concurrently downloading the same payload between different processes.
589 If locking fails the payload will be downloaded to a temporary file
590 separately for each process.
591 usable_globaltag_states (set(str)): Names of globaltag states accepted for
592 processing. This can be changed to make sure that only fully published
593 globaltags are used or to enable running on an open tag. It is not possible
594 to allow usage of 'INVALID' tags, those will always be rejected.
595 connection_timeout (int): timeout in seconds before connection should be
596 aborted. 0 sets the timeout to the default (300s)
597 stalled_timeout (int): timeout in seconds before a download should be
598 aborted if the speed stays below 10 KB/s, 0 disables this timeout
599 max_retries (int): maximum amount of retries if the server responded with
600 an HTTP response of 500 or more. 0 disables retrying
601 backoff_factor (int): backoff factor for retries in seconds. Retries are
602 performed using something similar to binary backoff: For retry :math:`n`
603 and a ``backoff_factor`` :math:`f` we wait for a random time chosen
604 uniformely from the interval :math:`[1, (2^{n} - 1) \times f]` in
605 seconds.
606)DOC")
607 .def("set_globaltag_callback", &Configuration::setGlobaltagCallbackPy, R"DOC(set_globaltag_callback(function)
608
609Set a callback function to be called just before processing.
610
611This callback can be used to further customize the globaltags to be used during
612processing. It will be called after the input files have been opened and checked
613with three keyword arguments:
614
615base_tags
616 The globaltags determined from either the input files or, if no input files
617 are present, the default globaltags
618
619user_tags
620 The globaltags provided by the user
621
622metadata
623 If there are not input files (e.g. generating events) this argument is None.
624 Otherwise it is a list of all the ``FileMetaData`` instances from all input files.
625 This list can be empty if there is no metadata associated with the input files.
626
627From this information the callback function should then compose the final list
628of globaltags to be used for processing and return this list. If ``None`` is
629returned the default behavior is applied as if there were no callback function.
630If anything else is returned the processing is aborted.
631
632If no callback function is specified the default behavior is equivalent to ::
633
634 def callback(base_tags, user_tags, metadata):
635 if not base_tags:
636 basf2.B2FATAL("No baseline globaltags available. Please use override")
637
638 return user_tags + base_tags
639
640If `override_enabled` is ``True`` then the callback function will not be called.
641
642Warning:
643 If a callback is set it is responsible to select the correct list of globaltags
644 and also make sure that all files are compatible. No further checks will be
645 done by the framework but any list of globaltags which is returned will be used
646 exactly as it is.
647
648 If the list of ``base_tags`` is empty that usually means that the input files
649 had different globaltag settings but it is the responsibility of the callback
650 to then verify if the list of globaltags is usable or not.
651
652 If the callback function determines that no working set of globaltags can be
653 determined then it should abort processing using a FATAL error or an exception
654)DOC")
655 ;
656
657 py::scope().attr("conditions") = py::ptr(&Configuration::getInstance());
658 }
659} // Belle2::Conditions namespace
bool m_overrideEnabled
is the globaltag override enabled?
CppOrPyList m_globalTags
the list with all user globaltags
void prependGlobalTag(const std::string &globalTag)
preprend a globaltag
Definition: Configuration.h:80
boost::python::tuple getDefaultGlobalTagsPy() const
Get the tuple of default globaltags as python version.
void appendGlobalTag(const std::string &globalTag)
Append a globaltag.
Definition: Configuration.h:78
Configuration()
Initialize default values.
void ensureEditable() const
Check whether the configuration object can be edited or if the database has been initialized already.
void setGlobaltagCallbackPy(const boost::python::object &obj)
Set a callback function from python which will be called when processing starts and should return the...
void disableGlobalTagReplay()
Disable global tag replay.
boost::python::list getGlobalTagsPy()
Get the list of user globaltags as python version.
Definition: Configuration.h:88
static Configuration & getInstance()
Get a reference to the instance which will be used when the Database is initialized.
void setMetadataProvidersPy(const boost::python::list &list)
Set the list of metadata providers in python.
boost::python::list getTestingPayloadLocationsPy()
Get the list of text files containing test payloads in python.
std::vector< std::string > getFinalListOfTags()
Get the final list of globaltags to be used for processing.
static void fillFromEnv(T &target, const std::string &envName, const std::string &defaultValue)
Fill a target object from a list of environment variables.
std::vector< FileMetaData > m_inputMetadata
the file metadata of all input files if globaltag replay is requested by input module
void setPayloadLocationsPy(const boost::python::list &list)
Set the list of payload locations in python.
CppOrPyList m_metadataProviders
the list with all the metadata providers
void setInputMetadata(const std::vector< FileMetaData > &inputMetadata)
To be called by input modules with the list of all input FileMetaData.
boost::python::list getMetadataProvidersPy()
Get the list of metadata providers in python.
void overrideGlobalTags()
Enable globaltag override: If this is called once than overrideEnabled() will return true and getFina...
std::vector< std::string > getBaseTags() const
Get the base globaltags to be used in addition to user globaltags.
void setTestingPayloadLocationsPy(const boost::python::list &list)
Set the list of text files containing test payloads in python.
std::optional< std::vector< std::string > > m_inputGlobaltags
the list of globaltags from all the input files to be used in addition to the user globaltags
void prependTestingPayloadLocation(const std::string &filename)
Prepend a local text file with testing payloads to the list.
boost::python::list getPayloadLocationsPy()
Get the list og payload locations in python.
static void exposePythonAPI()
expose this class to python
std::string m_defaultMetadataProviderUrl
default URL where to look for the metadata provider
std::string getDefaultMetadataProviderUrl()
Get the default URL where to look for the metadata provider.
CppOrPyList m_payloadLocations
the list with all the payload locations
std::vector< std::string > getDefaultGlobalTags() const
Get the std::vector of default globaltags.
void overrideGlobalTagsPy(const boost::python::list &globalTags)
Enable globaltag override and set the list of user globaltags in one go.
void reset()
Reset to default values.
void setGlobalTagsPy(const boost::python::list &globalTags)
Set the list of globaltags from python.
Definition: Configuration.h:84
bool overrideEnabled() const
Check if override is enabled by previous calls to overrideGlobalTags()
std::optional< boost::python::object > m_callback
the callback function to determine the final final list of globaltags
void appendTestingPayloadLocation(const std::string &filename)
Add a local text file with testing payloads.
bool m_databaseInitialized
bool indicating whether the database has been initialized, in which case any changes to the configura...
void prepend(const std::string &element)
Prepend an element to whatever representation we currently have.
void shallowCopy(const boost::python::object &source)
shallow copy all elements of the source object into the python representation.
void append(const std::string &element)
Append an element to whatever representation we currently have.
boost::python::list & ensurePy()
Return the python list version.
std::vector< std::string > & ensureCpp()
Return the C++ vector version.
std::variant< std::vector< std::string >, boost::python::list > m_value
Store either a std::vector or a python list of strings.
Definition: Configuration.h:45
static Downloader & getDefaultInstance()
Return the default instance.
Definition: Downloader.cc:134
static std::string get(const std::string &name, const std::string &fallback="")
Get the value of an environment variable or the given fallback value if the variable is not set.
static bool isSet(const std::string &name)
Check if a value is set in the database.
boost::python::object createROOTObjectPyCopy(const T &instance)
Create a python wrapped copy from a class instance which has a ROOT dictionary.
static Database & Instance()
Instance of a singleton Database.
Definition: Database.cc:42
static std::vector< std::string > getOrCreateList(const std::string &name, const std::string &fallback, const std::string &separators=" \t\n\r")
Get a list of values from an environment variable or the given fallback string if the variable is not...
static void reset(bool keepConfig=false)
Reset the database instance.
Definition: Database.cc:50
boost::python::object convertToPythonObject(const Scalar &value)
------------— From C++ TO Python Converter ---------------------—
bool iteratePythonObject(const boost::python::object &pyObject, Functor function)
Helper function to loop over a python object that implements the iterator concept and call a functor ...
Scalar convertPythonObject(const boost::python::object &pyObject, Scalar)
Convert from Python to given type.
static std::string name()
type name.