Belle II Software prerelease-11-00-00a
Configuration.cc
1/**************************************************************************
2 * basf2 (Belle II Analysis Software Framework) *
3 * Author: The Belle II Collaboration *
4 * *
5 * See git log for contributors and copyright holders. *
6 * This file is licensed under LGPL-3.0, see LICENSE.md. *
7 **************************************************************************/
8
9#include <framework/database/Configuration.h>
10#include <framework/logging/Logger.h>
11#include <framework/dataobjects/FileMetaData.h>
12#include <framework/database/Downloader.h>
13#include <framework/database/Database.h>
14#include <framework/utilities/Utils.h>
15#include <boost/python.hpp>
16#include <framework/core/PyObjConvUtils.h>
17#include <framework/core/PyObjROOTUtils.h>
18#include <boost/algorithm/string.hpp>
19
20#include <set>
21#include <regex>
22
23// Current default globaltag when generating events.
24#define CURRENT_DEFAULT_TAG "main_2026-04-29"
25
26namespace py = boost::python;
27
28namespace {
34 std::vector<std::string> extractStringList(const py::object& obj)
35 {
36 std::vector<std::string> result;
37 Belle2::PyObjConvUtils::iteratePythonObject(obj, [&result](const boost::python::object & item) {
38 py::object str(py::handle<>(PyObject_Str(item.ptr()))); // convert to string
39 py::extract<std::string> extract(str); // and extract
40 result.emplace_back(extract()); // and push back
41 return true;
42 });
43 // done, return
44 return result;
45 }
46}
47
48namespace Belle2::Conditions {
49 boost::python::list& CppOrPyList::ensurePy()
50 {
51 // convert to python list ...
52 if (m_value.index() == 0) {
53 boost::python::list tmp;
54 for (const auto& e : std::get<0>(m_value)) { tmp.append(e); }
55 m_value.emplace<boost::python::list>(std::move(tmp));
56 }
57 return std::get<1>(m_value);
58 }
59
60 std::vector<std::string>& CppOrPyList::ensureCpp()
61 {
62 // or convert to std::vector ...
63 if (m_value.index() == 1) {
64 std::vector<std::string> tmp = extractStringList(std::get<1>(m_value));
65 m_value.emplace<std::vector<std::string>>(std::move(tmp));
66 }
67 return std::get<0>(m_value);
68 }
69
70 void CppOrPyList::append(const std::string& element)
71 {
72 std::visit(Utils::VisitOverload{
73 [&element](std::vector<std::string>& list) {list.emplace_back(element);},
74 [&element](boost::python::list & list) {list.append(element);}
75 }, m_value);
76 }
77
78 void CppOrPyList::prepend(const std::string& element)
79 {
80 std::visit(Utils::VisitOverload{
81 [&element](std::vector<std::string>& list) {list.emplace(list.begin(), element);},
82 [&element](boost::python::list & list) {list.insert(0, element);}
83 }, m_value);
84 }
85
86 void CppOrPyList::shallowCopy(const boost::python::object& source)
87 {
88 ensurePy().slice(boost::python::_, boost::python::_) = source;
89 }
90
92 {
93 static Configuration instance;
94 return instance;
95 }
96
98 {
99 // Backwards compatibility with the existing BELLE2_CONDB_GLOBALTAG
100 // environment variable: If it is set disable replay
101 if (EnvironmentVariables::isSet("BELLE2_CONDB_GLOBALTAG")) {
102 fillFromEnv(m_globalTags, "BELLE2_CONDB_GLOBALTAG", "");
104 }
105 const std::string serverList = EnvironmentVariables::get("BELLE2_CONDB_SERVERLIST", "");
106 // The list of the metadata providers we are going to query:
107 const std::string metatadaProviders = serverList + " " + // First, the list of servers provided via env. variable
108 m_defaultLocalMetadataProviderPath + "/database.sqlite" + " " + // Then the default local provider (CVMFS)
109 m_defaultLegacyRemoteMetadataProviderServer + " " + // Then the Java-based legacy central provider
110 m_defaultHSFRemoteMetadataProviderServer; // Finally the HSF central provider
111 fillFromEnv(m_metadataProviders, "BELLE2_CONDB_METADATA", metatadaProviders);
113 }
114
116 {
119 }
120 *this = Configuration();
121 }
122
123 std::vector<std::string> Configuration::getDefaultGlobalTags() const
124 {
125 // currently the default globaltag can be overwritten by environment variable
126 // so keep that
127 return EnvironmentVariables::getOrCreateList("BELLE2_CONDB_GLOBALTAG", CURRENT_DEFAULT_TAG);
128 }
129
131 {
132 // same as above but as a python tuple ...
133 py::list list;
134 fillFromEnv(list, "BELLE2_CONDB_GLOBALTAG", CURRENT_DEFAULT_TAG);
135 return py::tuple(list);
136 }
137
138 void Configuration::setInputMetadata(const std::vector<FileMetaData>& inputMetadata)
139 {
141 m_inputMetadata = inputMetadata;
142 // make sure the list of globaltags to be used is created but empty
143 m_inputGlobaltags.emplace();
144 // now check for compatibility: make sure all metadata have the same globaltag
145 // setting. Unless we don't have metadata ...
146 if (inputMetadata.empty()) return;
147
148 std::optional<std::string> inputGlobaltags;
149 for (const auto& metadata : inputMetadata) {
150 if (!inputGlobaltags) {
151 inputGlobaltags = metadata.getDatabaseGlobalTag();
152 } else {
153 if (inputGlobaltags != metadata.getDatabaseGlobalTag()) {
154 B2WARNING("Input files metadata contain incompatible globaltag settings, globaltag replay not possible");
155 // no need to set anything
156 return;
157 }
158 }
159 }
160 // if it's still set and empty we have an empty input list ... warn specifically.
161 if (inputGlobaltags and inputGlobaltags->empty()) {
162 B2WARNING("Input files metadata all have empty globaltag setting, globaltag replay not possible");
163 return;
164 }
165 // set the list of globaltags from the string containing the globaltags
166 boost::split(*m_inputGlobaltags, *inputGlobaltags, boost::is_any_of(","));
167
168 // HACK: So, we successfully set the input globaltags from the input file,
169 // however we also decided that we want to add new payloads for
170 // boost, invariant mass, beam spot, collision axis in CMS.
171 // So if the release is older than when these features were introduced
172 // or if files were produced before specific date, extra GTs are appended.
173 // The appended GTs contain only the possible missing info and are added
174 // with lowest priority.
175 // If the files actually had all payloads these legacy payloads will never
176 // be used as they have lowest priority.
177 // Otherwise this should enable running over old files.
178 //
179 // TODO: Once we're sure all files being used contain all payloads remove this.
180
181 std::optional<std::string> relMin, dateMin;
182
183 for (const auto& metadata : inputMetadata) {
184 // get oldest release
185 std::string rel = metadata.getRelease().substr(0, 10);
186 if (std::regex_match(rel, std::regex("release-[0-9][0-9]"))) {
187 if (!relMin) relMin = rel;
188 relMin = min(*relMin, rel);
189 }
190
191 // get oldest production date
192 std::string date = metadata.getDate().substr(0, 10);
193 if (std::regex_match(date, std::regex("[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]"))) {
194 if (!dateMin) dateMin = date;
195 dateMin = min(*dateMin, date);
196 }
197 }
198
199 // add IP GT if rel older than rel04 or for old files
200 if ((relMin && relMin < "release-04") ||
201 (!relMin && (!dateMin || dateMin < "2019-12-31"))) {
202 B2DEBUG(30, "Enabling legacy IP information globaltag in tag replay");
203 m_inputGlobaltags->emplace_back("Legacy_IP_Information");
204 }
205
206 // add CollisionAxisCMS GT if rel older than rel08 or for old files
207 if ((relMin && relMin < "release-08") ||
208 (!relMin && (!dateMin || dateMin < "2023-08-31"))) {
209 B2DEBUG(30, "Enabling legacy CollsionAxisCMS globaltag in tag replay");
210 m_inputGlobaltags->emplace_back("Legacy_CollisionAxisCMS");
211 }
212 // END TODO/HACK
213 }
214
215 std::vector<std::string> Configuration::getBaseTags() const
216 {
217 // return the list of base tags to be used: Either the default tag
218 // or the list of globaltags from the input files
219 if (not m_inputGlobaltags) return getDefaultGlobalTags();
220 return *m_inputGlobaltags;
221 }
222
223 std::vector<std::string> Configuration::getFinalListOfTags()
224 {
225 if (m_overrideEnabled) {
226 B2INFO("Global tag override is in effect: input globaltags and default globaltag will be ignored");
227 return m_globalTags.ensureCpp();
228 }
229
230 auto baseList = getBaseTags();
231 if (m_callback) {
232 // Create a dictionary of keyword arguments for the callback
233 py::dict arguments;
234 // we want a python list of the base tags
235 {
236 py::list baseListPy;
237 for (const auto& tag : baseList) baseListPy.append(tag);
238 arguments["base_tags"] = baseListPy;
239 }
240 // and set the user tags from our list.
241 arguments["user_tags"] = m_globalTags.ensurePy();
242 // and prepare list of metadata. It's None when no replay has been
243 // requested which should mean that we generate events
244 arguments["metadata"] = py::object();
245 // otherwise it's a list of file metadata instances
246 if (m_inputGlobaltags) {
247 py::list metaDataList;
248 for (const auto& m : m_inputMetadata) metaDataList.append(createROOTObjectPyCopy(m));
249 arguments["metadata"] = metaDataList;
250 }
251 // arguments ready, call callback function, python will handle the exceptions
252 py::object retval = (*m_callback)(*py::tuple(), **arguments);
253 // If the return value is not None it should be an iterable
254 // containing the final tag list
255 if (retval != py::object()) {
256 return extractStringList(retval);
257 }
258 // callback returned None so fall back to default
259 }
260 // Default tag replay ... bail if list of globaltags is empty
261 if (baseList.empty()) {
262 if (m_inputGlobaltags) {
263 B2FATAL(R"(No baseline globaltags available.
264 The input files you selected don't have compatible globaltags or an empty
265 globaltag setting. As such globaltag configuration cannot be determined
266 automatically.
267
268 If you really sure that it is a good idea to process these files together
269 you have to manually override the list of globaltags:
270
271 >>> basf2.conditions.override_globaltags()
272)");
273 }else{
274 B2FATAL(R"(No default globaltags available.
275 There is no default globaltag available for processing. This usually means
276 you set the environment variable BELLE2_CONDB_GLOBALTAG to an empty value.
277
278 As this is unlikely to work for even the most basic functionality this is not
279 directly supported anymore. If you really want to disable any access to the
280 conditions database please configure this explicitly
281
282 >>> basf2.conditions.metadata_providers = []
283 >>> basf2.conditions.override_globaltags([])
284)");
285 }
286 }
287 // We have base tags and possibly user tags, so return both
288 std::vector finalList = m_globalTags.ensureCpp();
289 for (const auto& tag : baseList) { finalList.emplace_back(tag); }
290 return finalList;
291 }
292
293 namespace {
295 boost::python::dict expertSettings(const boost::python::tuple& args, boost::python::dict kwargs)
296 {
297 if (py::len(args) != 1) {
298 // keyword only function: raise typerror on non-keyword arguments
299 PyErr_SetString(PyExc_TypeError, ("expert_settings() takes one positional argument but " +
300 std::to_string(len(args)) + " were given").c_str());
301 py::throw_error_already_set();
302 }
303 Configuration& self = py::extract<Configuration&>(args[0]);
304
305 py::dict result;
306 // We want to check for a list of names if they exist in the input keyword
307 // arguments. If so, set the new value. In any case, add to output
308 // dictionary. Simplest way: create a variadic lambda with references to the
309 // dictionary and arguments for name, getter and setter.
310 auto checkValue = [&kwargs, &result](const std::string & name, auto setter, auto getter) {
311 using value_type = decltype(getter());
312 if (kwargs.has_key(name)) {
313 value_type value{};
314 py::object object = kwargs[name];
315 try {
316 value = PyObjConvUtils::convertPythonObject(object, value);
317 } catch (std::runtime_error&) {
318 std::stringstream error;
319 error << "Cannot convert argument '" << name << "' to " << PyObjConvUtils::Type<value_type>::name();
320 PyErr_SetString(PyExc_TypeError, error.str().c_str());
321 py::throw_error_already_set();
322 }
323 setter(value);
324 // remove key from kwargs so we can easily check for ones we don't understand later
325 py::delitem(kwargs, py::object(name));
326 }
327 result[name] = PyObjConvUtils::convertToPythonObject(getter());
328 };
329 auto& downloader = Downloader::getDefaultInstance();
330 // That was all the heavy lifting, now just declare all known options :D
331 // I would love to indent this a bit better but astyle has different opinions
332 checkValue("save_payloads",
333 [&self](const std::string & path) { self.setNewPayloadLocation(path);},
334 [&self]() {return self.getNewPayloadLocation();});
335 checkValue("download_cache_location",
336 [&self](const std::string & path) { self.setDownloadCacheDirectory(path);},
337 [&self]() {return self.getDownloadCacheDirectory();});
338 checkValue("download_lock_timeout",
339 [&self](size_t timeout) { self.setDownloadLockTimeout(timeout);},
340 [&self]() { return self.getDownloadLockTimeout();});
341 checkValue("usable_globaltag_states",
342 [&self](const auto & states) { self.setUsableTagStates(states); },
343 [&self]() { return self.getUsableTagStates(); });
344 checkValue("connection_timeout",
345 [&downloader](unsigned int timeout) {downloader.setConnectionTimeout(timeout);},
346 [&downloader]() { return downloader.getConnectionTimeout();});
347 checkValue("stalled_timeout",
348 [&downloader](unsigned int timeout) {downloader.setStalledTimeout(timeout);},
349 [&downloader]() { return downloader.getStalledTimeout();});
350 checkValue("max_retries",
351 [&downloader](unsigned int retries) {downloader.setMaxRetries(retries);},
352 [&downloader]() { return downloader.getMaxRetries();});
353 checkValue("backoff_factor",
354 [&downloader](unsigned int factor) { downloader.setBackoffFactor(factor);},
355 [&downloader]() { return downloader.getBackoffFactor();});
356 // And lastly check if there is something in the kwargs we don't understand ...
357 if (py::len(kwargs) > 0) {
358 std::string message = "Unrecognized keyword arguments: ";
359 auto keys = kwargs.keys();
360 for (int i = 0; i < len(keys); ++i) {
361 if (i > 0) message += ", ";
362 message += py::extract<std::string>(keys[i]);
363 }
364 PyErr_SetString(PyExc_TypeError, message.c_str());
365 py::throw_error_already_set();
366 }
367 return result;
368 }
369 }
370
371 void Configuration::overrideGlobalTagsPy(const boost::python::list& globalTags)
372 {
373 setGlobalTagsPy(globalTags);
374 m_overrideEnabled = true;
375 }
376
378 {
379 //don't show c++ signature in python doc to keep it simple
380 py::docstring_options options(true, false, false);
381
382 void (Configuration::*overrideGTFlag)() = &Configuration::overrideGlobalTags;
383 void (Configuration::*overrideGTList)(const py::list&) = &Configuration::overrideGlobalTagsPy;
384 py::object expert = raw_function(expertSettings);
385 py::class_<Configuration>("ConditionsConfiguration", R"DOC(
386This class contains all configurations for the conditions database service
387
388* which globaltags to use
389* where to look for payload information
390* where to find the actual payload files
391* which temporary testing payloads to use
392
393But for most users the only thing they should need to care about is to set the
394list of additional `globaltags` to use.
395)DOC")
396 .add_property("override_enabled", &Configuration::overrideEnabled, R"DOC(
397Indicator whether or not the override of globaltags is enabled. If true then
398globaltags present in input files will be ignored and only the ones given in
399`globaltags` will be considered.
400)DOC")
401 .def("reset", &Configuration::reset, R"DOC(reset()
402
403Reset the conditions database configuration to its original state.
404)DOC")
405 .add_property("default_globaltags", &Configuration::getDefaultGlobalTagsPy, R"DOC(
406A tuple containing the default globaltags to be used if events are generated without an input file.
407)DOC")
408 .add_property("globaltags", &Configuration::getGlobalTagsPy, &Configuration::setGlobalTagsPy, R"DOC(
409List of globaltags to be used. These globaltags will be the ones with highest
410priority but by default the globaltags used to create the input files or the
411default globaltag will also be used.
412
413The priority of the globaltags in this list is highest first. So the first in
414the list will be checked first and all other globaltags will only be checked for
415payloads not found so far.
416
417Warning:
418 By default this list contains the globaltags to be used **in addition** to
419 the ones from the input file or the default one if no input file is present.
420 If this is not desirable you need to call `override_globaltags()` to disable
421 any addition or modification of this list.
422)DOC")
423 .def("append_globaltag", &Configuration::appendGlobalTag, py::args("name"), R"DOC(append_globaltag(name)
424
425Append a globaltag to the end of the `globaltags` list. That means it will be
426the lowest priority of all tags in the list.
427)DOC")
428 .def("prepend_globaltag", &Configuration::prependGlobalTag, py::args("name"), R"DOC(prepend_globaltag(name)
429
430Add a globaltag to the beginning of the `globaltags` list. That means it will be
431the highest priority of all tags in the list.
432)DOC")
433 .def("override_globaltags", overrideGTFlag)
434 .def("override_globaltags", overrideGTList, py::args("globaltags"), R"DOC(override_globaltags(list=None)
435
436Enable globaltag override. This disables all modification of the globaltag list at the beginning of processing:
437
438* the default globaltag or the input file globaltags will be ignored.
439* any callback set with `set_globaltag_callback` will be ignored.
440* the list of `globaltags` will be used exactly as it is.
441
442Parameters:
443 list (list(str) or None) if given this list will replace the current content of `globaltags`
444
445Warning:
446 it's still possible to modify `globaltags` after this call.
447)DOC")
448 .def("disable_globaltag_replay", &Configuration::disableGlobalTagReplay, R"DOC(disable_globaltag_replay()
449
450Disable global tag replay and revert to the old behavior that the default
451globaltag will be used if no other globaltags are specified.
452
453This is a shortcut to just calling
454
455 >>> conditions.override_globaltags()
456 >>> conditions.globaltags += list(conditions.default_globaltags)
457
458)DOC")
459 .def("append_testing_payloads", &Configuration::appendTestingPayloadLocation, py::args("filename"), R"DOC(append_testing_payloads(filename)
460
461Append a text file containing local test payloads to the end of the list of
462`testing_payloads`. This will mean they will have lower priority than payloads
463in previously defined text files but still higher priority than globaltags.
464
465Parameters:
466 filename (str): file containing a local definition of payloads and their
467 intervals of validity for testing
468
469Warning:
470 This functionality is strictly for testing purposes. Using local payloads
471 leads to results which cannot be reproduced by anyone else and thus cannot
472 be published.
473)DOC")
474 .def("prepend_testing_payloads", &Configuration::prependTestingPayloadLocation, py::args("filename"), R"DOC(prepend_testing_payloads(filename)
475
476Insert a text file containing local test payloads in the beginning of the list
477of `testing_payloads`. This will mean they will have higher priority than payloads in
478previously defined text files as well as higher priority than globaltags.
479
480Parameters:
481 filename (str): file containing a local definition of payloads and their
482 intervals of validity for testing
483
484Warning:
485 This functionality is strictly for testing purposes. Using local payloads
486 leads to results which cannot be reproduced by anyone else and thus cannot
487 be published.
488)DOC")
490List of text files to look for local testing payloads. Each entry should be a
491text file containing local payloads and their intervals of validity to be used
492for testing.
493
494Payloads found in these files and valid for the current run will have a higher
495priority than any of the `globaltags`. If a valid payload is present in multiple
496files the first one in the list will have higher priority.
497
498Warning:
499 This functionality is strictly for testing purposes. Using local payloads
500 leads to results which cannot be reproduced by anyone else and thus cannot
501 be published.
502)DOC")
503 .add_property("metadata_providers", &Configuration::getMetadataProvidersPy, &Configuration::setMetadataProvidersPy, R"DOC(
504List of metadata providers to use when looking for payload metadata. There are currently two supported providers:
505
5061. Central metadata provider to look for payloads in the central conditions database.
507 This provider is used for any entry in this list which starts with ``http(s)://``.
508 The URL should point to the top level of the REST api endpoints on the server
509
5102. Local metadata provider to look for payloads in a local SQLite snapshot taken
511 from the central server. This provider will be assumed for any entry in this
512 list not starting with a protocol specifier or if the protocol is given as ``file://``
513
514This list should rarely need to be changed. The only exception is for users who
515want to be able to use the software without internet connection after they
516downloaded a snapshot of the necessary globaltags with ``b2conditionsdb download``
517to point to this location.
518)DOC")
519 .add_property("default_metadata_provider_server", &Configuration::getDefaultRemoteMetadataProviderServer, R"DOC(
520URL of the default central metadata provider to look for payloads in the
521conditions database.
522)DOC")
523 .add_property("default_hsf_metadata_provider_server", &Configuration::getDefaultHSFRemoteMetadataProviderServer, R"DOC(
524URL of the default HSF central metadata provider to look for payloads in the
525conditions database.
526)DOC")
527 .add_property("payload_locations", &Configuration::getPayloadLocationsPy, &Configuration::setPayloadLocationsPy, R"DOC(
528List of payload locations to search for payloads which have been found by any of
529the configured `metadata_providers`. This can be a local directory or a
530``http(s)://`` url pointing to the payload directory on a server.
531
532For remote locations starting with ``http(s)://`` we assume that the layout of
533the payloads on the server is the same as on the main payload server:
534The combination of given location and the relative url in the payload metadata
535field ``payloadUrl`` should point to the correct payload on the server.
536
537For local directories, two layouts are supported and will be auto detected:
538
539flat
540 All payloads are in the same directory without any substructure with the name
541 ``dbstore_{name}_rev_{revision}.root``
542hashed
543 All payloads are stored in subdirectories in the form ``AB/{name}_r{revision}.root``
544 where ``A`` and ``B`` are the first two characters of the md5 checksum of the
545 payload file.
546
547Example:
548 Given ``payload_locations = ["payload_dir/", "http://server.com/payloads"]``
549 the framework would look for a payload with name ``BeamParameters`` in revision
550 ``45`` (and checksum ``a34ce5...``) in the following places
551
552
553 1. ``payload_dir/a3/BeamParameters_r45.root``
554 2. ``payload_dir/dbstore_BeamParameters_rev_45.root``
555 3. ``http://server.com/payloads/dbstore/BeamParameters/dbstore_BeamParameters_rev_45.root``
556 given the usual pattern of the ``payloadUrl`` metadata. But this could be
557 changed on the central servers so mirrors should not depend on this convention
558 but copy the actual structure of the central server.
559
560If the payload cannot be found in any of the given locations the framework will
561always attempt to download it directly from the central server and put it in a
562local cache directory.
563)DOC")
564 .def("expert_settings", expert, R"DOC(expert_settings(**kwargs)
565
566Set some additional settings for the conditions database.
567
568You can supply any combination of keyword-only arguments defined below. The
569function will return a dictionary containing all current settings.
570
571 >>> conditions.expert_settings(connection_timeout=5, max_retries=1)
572 {'save_payloads': 'localdb/database.txt',
573 'download_cache_location': '',
574 'download_lock_timeout': 120,
575 'usable_globaltag_states': {'PUBLISHED', 'RUNNING', 'TESTING', 'VALIDATED'},
576 'connection_timeout': 5,
577 'stalled_timeout': 60,
578 'max_retries': 1,
579 'backoff_factor': 5}
580
581Warning:
582 Modification of these parameters should not be needed, in rare
583 circumstances this could be used to optimize access for many jobs at once
584 but should only be set by experts.
585
586Parameters:
587 save_payloads (str): Where to store new payloads created during processing.
588 This should be a filename to contain the payload information and the payload
589 files will be placed in the same directory as the file.
590 download_cache_location (str): Where to store payloads which have been downloaded
591 from the central server. This could be a user defined directory, otherwise
592 empty string defaults to ``$TMPDIR/basf2-conditions`` where ``$TMPDIR`` is the
593 temporary directories defined in the system. Newly downloaded payloads will
594 be stored in this directory in a hashed structure, see `payload_locations`
595 download_lock_timeout (int): How many seconds to wait for a write lock when
596 concurrently downloading the same payload between different processes.
597 If locking fails the payload will be downloaded to a temporary file
598 separately for each process.
599 usable_globaltag_states (set(str)): Names of globaltag states accepted for
600 processing. This can be changed to make sure that only fully published
601 globaltags are used or to enable running on an open tag. It is not possible
602 to allow usage of 'INVALID' tags, those will always be rejected.
603 connection_timeout (int): timeout in seconds before connection should be
604 aborted. 0 sets the timeout to the default (300s)
605 stalled_timeout (int): timeout in seconds before a download should be
606 aborted if the speed stays below 10 KB/s, 0 disables this timeout
607 max_retries (int): maximum amount of retries if the server responded with
608 an HTTP response of 500 or more. 0 disables retrying
609 backoff_factor (int): backoff factor for retries in seconds. Retries are
610 performed using something similar to binary backoff: For retry :math:`n`
611 and a ``backoff_factor`` :math:`f` we wait for a random time chosen
612 uniformly from the interval :math:`[1, (2^{n} - 1) \times f]` in
613 seconds.
614)DOC")
615 .def("set_globaltag_callback", &Configuration::setGlobaltagCallbackPy, R"DOC(set_globaltag_callback(function)
616
617Set a callback function to be called just before processing.
618
619This callback can be used to further customize the globaltags to be used during
620processing. It will be called after the input files have been opened and checked
621with three keyword arguments:
622
623base_tags
624 The globaltags determined from either the input files or, if no input files
625 are present, the default globaltags
626
627user_tags
628 The globaltags provided by the user
629
630metadata
631 If there are not input files (e.g. generating events) this argument is None.
632 Otherwise it is a list of all the ``FileMetaData`` instances from all input files.
633 This list can be empty if there is no metadata associated with the input files.
634
635From this information the callback function should then compose the final list
636of globaltags to be used for processing and return this list. If ``None`` is
637returned the default behavior is applied as if there were no callback function.
638If anything else is returned the processing is aborted.
639
640If no callback function is specified the default behavior is equivalent to ::
641
642 def callback(base_tags, user_tags, metadata):
643 if not base_tags:
644 basf2.B2FATAL("No baseline globaltags available. Please use override")
645
646 return user_tags + base_tags
647
648If `override_enabled` is ``True`` then the callback function will not be called.
649
650Warning:
651 If a callback is set it is responsible to select the correct list of globaltags
652 and also make sure that all files are compatible. No further checks will be
653 done by the framework but any list of globaltags which is returned will be used
654 exactly as it is.
655
656 If the list of ``base_tags`` is empty that usually means that the input files
657 had different globaltag settings but it is the responsibility of the callback
658 to then verify if the list of globaltags is usable or not.
659
660 If the callback function determines that no working set of globaltags can be
661 determined then it should abort processing using a FATAL error or an exception
662)DOC")
663 ;
664
665 py::scope().attr("conditions") = py::ptr(&Configuration::getInstance());
666 }
667} // Belle2::Conditions namespace
std::string getDefaultHSFRemoteMetadataProviderServer()
Get the default server URL for the HSF central metadata provider.
bool m_overrideEnabled
is the globaltag override enabled?
CppOrPyList m_globalTags
the list with all user globaltags
void prependGlobalTag(const std::string &globalTag)
prepend a globaltag
std::string m_defaultHSFRemoteMetadataProviderServer
default server URL for the HSF remote metadata provider
void appendGlobalTag(const std::string &globalTag)
Append a globaltag.
Configuration()
Initialize default values.
void ensureEditable() const
Check whether the configuration object can be edited or if the database has been initialized already.
void setGlobaltagCallbackPy(const boost::python::object &obj)
Set a callback function from python which will be called when processing starts and should return the...
void disableGlobalTagReplay()
Disable global tag replay.
boost::python::list getGlobalTagsPy()
Get the list of user globaltags as python version.
void setMetadataProvidersPy(const boost::python::list &list)
Set the list of metadata providers in python.
boost::python::list getTestingPayloadLocationsPy()
Get the list of text files containing test payloads in python.
std::string m_defaultLocalMetadataProviderPath
default local path for the local metadata provider
std::vector< std::string > getFinalListOfTags()
Get the final list of globaltags to be used for processing.
boost::python::tuple getDefaultGlobalTagsPy() const
Get the tuple of default globaltags as python version.
static void fillFromEnv(T &target, const std::string &envName, const std::string &defaultValue)
Fill a target object from a list of environment variables.
std::vector< FileMetaData > m_inputMetadata
the file metadata of all input files if globaltag replay is requested by input module
static Configuration & getInstance()
Get a reference to the instance which will be used when the Database is initialized.
void setPayloadLocationsPy(const boost::python::list &list)
Set the list of payload locations in python.
CppOrPyList m_metadataProviders
the list with all the metadata providers
void setInputMetadata(const std::vector< FileMetaData > &inputMetadata)
To be called by input modules with the list of all input FileMetaData.
boost::python::list getMetadataProvidersPy()
Get the list of metadata providers in python.
void overrideGlobalTags()
Enable globaltag override: If this is called once than overrideEnabled() will return true and getFina...
std::vector< std::string > getBaseTags() const
Get the base globaltags to be used in addition to user globaltags.
std::string m_defaultLegacyRemoteMetadataProviderServer
default server URL for the (legacy) remote metadata provider
void setTestingPayloadLocationsPy(const boost::python::list &list)
Set the list of text files containing test payloads in python.
static void exposePythonAPI()
expose this class to python
std::optional< std::vector< std::string > > m_inputGlobaltags
the list of globaltags from all the input files to be used in addition to the user globaltags
void prependTestingPayloadLocation(const std::string &filename)
Prepend a local text file with testing payloads to the list.
boost::python::list getPayloadLocationsPy()
Get the list og payload locations in python.
CppOrPyList m_payloadLocations
the list with all the payload locations
std::vector< std::string > getDefaultGlobalTags() const
Get the std::vector of default globaltags.
void overrideGlobalTagsPy(const boost::python::list &globalTags)
Enable globaltag override and set the list of user globaltags in one go.
void reset()
Reset to default values.
std::string getDefaultRemoteMetadataProviderServer()
Get the default server URL for the remote metadata provider.
void setGlobalTagsPy(const boost::python::list &globalTags)
Set the list of globaltags from python.
bool overrideEnabled() const
Check if override is enabled by previous calls to overrideGlobalTags()
std::optional< boost::python::object > m_callback
the callback function to determine the final final list of globaltags
void appendTestingPayloadLocation(const std::string &filename)
Add a local text file with testing payloads.
bool m_databaseInitialized
bool indicating whether the database has been initialized, in which case any changes to the configura...
void prepend(const std::string &element)
Prepend an element to whatever representation we currently have.
void shallowCopy(const boost::python::object &source)
shallow copy all elements of the source object into the python representation.
void append(const std::string &element)
Append an element to whatever representation we currently have.
boost::python::list & ensurePy()
Return the python list version.
std::vector< std::string > & ensureCpp()
Return the C++ vector version.
std::variant< std::vector< std::string >, boost::python::list > m_value
Store either a std::vector or a python list of strings.
static Downloader & getDefaultInstance()
Return the default instance.
static std::string get(const std::string &name, const std::string &fallback="")
Get the value of an environment variable or the given fallback value if the variable is not set.
static bool isSet(const std::string &name)
Check if a value is set in the database.
boost::python::object createROOTObjectPyCopy(const T &instance)
Create a python wrapped copy from a class instance which has a ROOT dictionary.
static Database & Instance()
Instance of a singleton Database.
Definition Database.cc:42
static std::vector< std::string > getOrCreateList(const std::string &name, const std::string &fallback, const std::string &separators=" \t\n\r")
Get a list of values from an environment variable or the given fallback string if the variable is not...
static void reset(bool keepConfig=false)
Reset the database instance.
Definition Database.cc:50
boost::python::object convertToPythonObject(const Scalar &value)
------------— From C++ TO Python Converter ---------------------—
bool iteratePythonObject(const boost::python::object &pyObject, Functor function)
Helper function to loop over a python object that implements the iterator concept and call a functor ...
Scalar convertPythonObject(const boost::python::object &pyObject, Scalar)
Convert from Python to given type.
static std::string name()
type name.
Helper struct for the C++17 std::visit overload pattern to allow simple use of variants.
Definition Utils.h:26