9#include <framework/database/Configuration.h>
10#include <framework/logging/Logger.h>
11#include <framework/dataobjects/FileMetaData.h>
12#include <framework/database/Downloader.h>
13#include <framework/database/Database.h>
14#include <framework/utilities/Utils.h>
15#include <boost/python.hpp>
16#include <framework/core/PyObjConvUtils.h>
17#include <framework/core/PyObjROOTUtils.h>
18#include <boost/algorithm/string.hpp>
24#define CURRENT_DEFAULT_TAG "main_2025-07-22"
26namespace py = boost::python;
34 std::vector<std::string> extractStringList(
const py::object& obj)
36 std::vector<std::string> result;
38 py::object str(py::handle<>(PyObject_Str(item.ptr())));
39 py::extract<std::string> extract(str);
40 result.emplace_back(extract());
48namespace Belle2::Conditions {
53 boost::python::list tmp;
54 for (
const auto& e : std::get<0>(
m_value)) { tmp.append(e); }
55 m_value.emplace<boost::python::list>(std::move(tmp));
64 std::vector<std::string> tmp = extractStringList(std::get<1>(
m_value));
65 m_value.emplace<std::vector<std::string>>(std::move(tmp));
73 [&element](std::vector<std::string>& list) {list.emplace_back(element);},
74 [&element](boost::python::list & list) {list.append(element);}
81 [&element](std::vector<std::string>& list) {list.emplace(list.begin(), element);},
82 [&element](boost::python::list & list) {list.insert(0, element);}
88 ensurePy().slice(boost::python::_, boost::python::_) = source;
107 const std::string metatadaProviders = serverList +
" " +
133 fillFromEnv(list,
"BELLE2_CONDB_GLOBALTAG", CURRENT_DEFAULT_TAG);
134 return py::tuple(list);
145 if (inputMetadata.empty())
return;
147 std::optional<std::string> inputGlobaltags;
148 for (
const auto& metadata : inputMetadata) {
149 if (!inputGlobaltags) {
150 inputGlobaltags = metadata.getDatabaseGlobalTag();
152 if (inputGlobaltags != metadata.getDatabaseGlobalTag()) {
153 B2WARNING(
"Input files metadata contain incompatible globaltag settings, globaltag replay not possible");
160 if (inputGlobaltags and inputGlobaltags->empty()) {
161 B2WARNING(
"Input files metadata all have empty globaltag setting, globaltag replay not possible");
180 std::optional<std::string> relMin, dateMin;
182 for (
const auto& metadata : inputMetadata) {
184 std::string rel = metadata.getRelease().substr(0, 10);
185 if (std::regex_match(rel, std::regex(
"release-[0-9][0-9]"))) {
186 if (!relMin) relMin = rel;
187 relMin = min(*relMin, rel);
191 std::string date = metadata.getDate().substr(0, 10);
192 if (std::regex_match(date, std::regex(
"[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]"))) {
193 if (!dateMin) dateMin = date;
194 dateMin = min(*dateMin, date);
199 if ((relMin && relMin <
"release-04") ||
200 (!relMin && (!dateMin || dateMin <
"2019-12-31"))) {
201 B2DEBUG(30,
"Enabling legacy IP information globaltag in tag replay");
206 if ((relMin && relMin <
"release-08") ||
207 (!relMin && (!dateMin || dateMin <
"2023-08-31"))) {
208 B2DEBUG(30,
"Enabling legacy CollsionAxisCMS globaltag in tag replay");
225 B2INFO(
"Global tag override is in effect: input globaltags and default globaltag will be ignored");
236 for (
const auto& tag : baseList) baseListPy.append(tag);
237 arguments[
"base_tags"] = baseListPy;
243 arguments[
"metadata"] = py::object();
246 py::list metaDataList;
248 arguments[
"metadata"] = metaDataList;
251 py::object retval = (*m_callback)(*py::tuple(), **arguments);
254 if (retval != py::object()) {
255 return extractStringList(retval);
260 if (baseList.empty()) {
262 B2FATAL(R
"(No baseline globaltags available.
263 The input files you selected don't have compatible globaltags or an empty
264 globaltag setting. As such globaltag configuration cannot be determined
267 If you really sure that it is a good idea to process these files together
268 you have to manually override the list of globaltags:
270 >>> basf2.conditions.override_globaltags()
273 B2FATAL(R
"(No default globaltags available.
274 There is no default globaltag available for processing. This usually means
275 you set the environment variable BELLE2_CONDB_GLOBALTAG to an empty value.
277 As this is unlikely to work for even the most basic functionality this is not
278 directly supported anymore. If you really want to disable any access to the
279 conditions database please configure this explicitly
281 >>> basf2.conditions.metadata_providers = []
282 >>> basf2.conditions.override_globaltags([])
288 for (
const auto& tag : baseList) { finalList.emplace_back(tag); }
294 boost::python::dict expertSettings(
const boost::python::tuple& args, boost::python::dict kwargs)
296 if (py::len(args) != 1) {
298 PyErr_SetString(PyExc_TypeError, (
"expert_settings() takes one positional argument but " +
299 std::to_string(len(args)) +
" were given").c_str());
300 py::throw_error_already_set();
302 Configuration& self = py::extract<Configuration&>(args[0]);
309 auto checkValue = [&kwargs, &result](
const std::string & name,
auto setter,
auto getter) {
310 using value_type =
decltype(getter());
311 if (kwargs.has_key(name)) {
313 py::object
object = kwargs[name];
316 }
catch (std::runtime_error&) {
317 std::stringstream error;
319 PyErr_SetString(PyExc_TypeError, error.str().c_str());
320 py::throw_error_already_set();
324 py::delitem(kwargs, py::object(name));
331 checkValue(
"save_payloads",
332 [&self](
const std::string & path) { self.setNewPayloadLocation(path);},
333 [&self]() {
return self.getNewPayloadLocation();});
334 checkValue(
"download_cache_location",
335 [&self](
const std::string & path) { self.setDownloadCacheDirectory(path);},
336 [&self]() {
return self.getDownloadCacheDirectory();});
337 checkValue(
"download_lock_timeout",
338 [&self](
size_t timeout) { self.setDownloadLockTimeout(timeout);},
339 [&self]() {
return self.getDownloadLockTimeout();});
340 checkValue(
"usable_globaltag_states",
341 [&self](
const auto & states) { self.setUsableTagStates(states); },
342 [&self]() {
return self.getUsableTagStates(); });
343 checkValue(
"connection_timeout",
344 [&downloader](
unsigned int timeout) {downloader.setConnectionTimeout(timeout);},
345 [&downloader]() {
return downloader.getConnectionTimeout();});
346 checkValue(
"stalled_timeout",
347 [&downloader](
unsigned int timeout) {downloader.setStalledTimeout(timeout);},
348 [&downloader]() {
return downloader.getStalledTimeout();});
349 checkValue(
"max_retries",
350 [&downloader](
unsigned int retries) {downloader.setMaxRetries(retries);},
351 [&downloader]() {
return downloader.getMaxRetries();});
352 checkValue(
"backoff_factor",
353 [&downloader](
unsigned int factor) { downloader.setBackoffFactor(factor);},
354 [&downloader]() {
return downloader.getBackoffFactor();});
356 if (py::len(kwargs) > 0) {
357 std::string message =
"Unrecognized keyword arguments: ";
358 auto keys = kwargs.keys();
359 for (
int i = 0; i < len(keys); ++i) {
360 if (i > 0) message +=
", ";
361 message += py::extract<std::string>(keys[i]);
363 PyErr_SetString(PyExc_TypeError, message.c_str());
364 py::throw_error_already_set();
379 py::docstring_options options(
true,
false,
false);
383 py::object expert = raw_function(expertSettings);
384 py::class_<Configuration>(
"ConditionsConfiguration", R
"DOC(
385This class contains all configurations for the conditions database service
387* which globaltags to use
388* where to look for payload information
389* where to find the actual payload files
390* which temporary testing payloads to use
392But for most users the only thing they should need to care about is to set the
393list of additional `globaltags` to use.
396Indicator whether or not the override of globaltags is enabled. If true then
397globaltags present in input files will be ignored and only the ones given in
398`globaltags` will be considered.
402Reset the conditions database configuration to its original state.
405A tuple containing the default globaltags to be used if events are generated without an input file.
408List of globaltags to be used. These globaltags will be the ones with highest
409priority but by default the globaltags used to create the input files or the
410default globaltag will also be used.
412The priority of the globaltags in this list is highest first. So the first in
413the list will be checked first and all other globaltags will only be checked for
414payloads not found so far.
417 By default this list contains the globaltags to be used **in addition** to
418 the ones from the input file or the default one if no input file is present.
419 If this is not desirable you need to call `override_globaltags()` to disable
420 any addition or modification of this list.
424Append a globaltag to the end of the `globaltags` list. That means it will be
425the lowest priority of all tags in the list.
429Add a globaltag to the beginning of the `globaltags` list. That means it will be
430the highest priority of all tags in the list.
432 .def("override_globaltags", overrideGTFlag)
433 .def(
"override_globaltags", overrideGTList, py::args(
"globaltags"), R
"DOC(override_globaltags(list=None)
435Enable globaltag override. This disables all modification of the globaltag list at the beginning of processing:
437* the default globaltag or the input file globaltags will be ignored.
438* any callback set with `set_globaltag_callback` will be ignored.
439* the list of `globaltags` will be used exactly as it is.
442 list (list(str) or None) if given this list will replace the current content of `globaltags`
445 it's still possible to modify `globaltags` after this call.
449Disable global tag replay and revert to the old behavior that the default
450globaltag will be used if no other globaltags are specified.
452This is a shortcut to just calling
454 >>> conditions.override_globaltags()
455 >>> conditions.globaltags += list(conditions.default_globaltags)
460Append a text file containing local test payloads to the end of the list of
461`testing_payloads`. This will mean they will have lower priority than payloads
462in previously defined text files but still higher priority than globaltags.
465 filename (str): file containing a local definition of payloads and their
466 intervals of validity for testing
469 This functionality is strictly for testing purposes. Using local payloads
470 leads to results which cannot be reproduced by anyone else and thus cannot
475Insert a text file containing local test payloads in the beginning of the list
476of `testing_payloads`. This will mean they will have higher priority than payloads in
477previously defined text files as well as higher priority than globaltags.
480 filename (str): file containing a local definition of payloads and their
481 intervals of validity for testing
484 This functionality is strictly for testing purposes. Using local payloads
485 leads to results which cannot be reproduced by anyone else and thus cannot
489List of text files to look for local testing payloads. Each entry should be a
490text file containing local payloads and their intervals of validity to be used
493Payloads found in these files and valid for the current run will have a higher
494priority than any of the `globaltags`. If a valid payload is present in multiple
495files the first one in the list will have higher priority.
498 This functionality is strictly for testing purposes. Using local payloads
499 leads to results which cannot be reproduced by anyone else and thus cannot
503List of metadata providers to use when looking for payload metadata. There are currently two supported providers:
5051. Central metadata provider to look for payloads in the central conditions database.
506 This provider is used for any entry in this list which starts with ``http(s)://``.
507 The URL should point to the top level of the REST api endpoints on the server
5092. Local metadata provider to look for payloads in a local SQLite snapshot taken
510 from the central server. This provider will be assumed for any entry in this
511 list not starting with a protocol specifier or if the protocol is given as ``file://``
513This list should rarely need to be changed. The only exception is for users who
514want to be able to use the software without internet connection after they
515downloaded a snapshot of the necessary globaltags with ``b2conditionsdb download``
516to point to this location.
519URL of the default central metadata provider to look for payloads in the
523List of payload locations to search for payloads which have been found by any of
524the configured `metadata_providers`. This can be a local directory or a
525``http(s)://`` url pointing to the payload directory on a server.
527For remote locations starting with ``http(s)://`` we assume that the layout of
528the payloads on the server is the same as on the main payload server:
529The combination of given location and the relative url in the payload metadata
530field ``payloadUrl`` should point to the correct payload on the server.
532For local directories, two layouts are supported and will be auto detected:
535 All payloads are in the same directory without any substructure with the name
536 ``dbstore_{name}_rev_{revision}.root``
538 All payloads are stored in subdirectories in the form ``AB/{name}_r{revision}.root``
539 where ``A`` and ``B`` are the first two characters of the md5 checksum of the
543 Given ``payload_locations = ["payload_dir/", "http://server.com/payloads"]``
544 the framework would look for a payload with name ``BeamParameters`` in revision
545 ``45`` (and checksum ``a34ce5...``) in the following places
548 1. ``payload_dir/a3/BeamParameters_r45.root``
549 2. ``payload_dir/dbstore_BeamParameters_rev_45.root``
550 3. ``http://server.com/payloads/dbstore/BeamParameters/dbstore_BeamParameters_rev_45.root``
551 given the usual pattern of the ``payloadUrl`` metadata. But this could be
552 changed on the central servers so mirrors should not depend on this convention
553 but copy the actual structure of the central server.
555If the payload cannot be found in any of the given locations the framework will
556always attempt to download it directly from the central server and put it in a
557local cache directory.
559 .def("expert_settings", expert, R
"DOC(expert_settings(**kwargs)
561Set some additional settings for the conditions database.
563You can supply any combination of keyword-only arguments defined below. The
564function will return a dictionary containing all current settings.
566 >>> conditions.expert_settings(connection_timeout=5, max_retries=1)
567 {'save_payloads': 'localdb/database.txt',
568 'download_cache_location': '',
569 'download_lock_timeout': 120,
570 'usable_globaltag_states': {'PUBLISHED', 'RUNNING', 'TESTING', 'VALIDATED'},
571 'connection_timeout': 5,
572 'stalled_timeout': 60,
577 Modification of these parameters should not be needed, in rare
578 circumstances this could be used to optimize access for many jobs at once
579 but should only be set by experts.
582 save_payloads (str): Where to store new payloads created during processing.
583 This should be a filename to contain the payload information and the payload
584 files will be placed in the same directory as the file.
585 download_cache_location (str): Where to store payloads which have been downloaded
586 from the central server. This could be a user defined directory, otherwise
587 empty string defaults to ``$TMPDIR/basf2-conditions`` where ``$TMPDIR`` is the
588 temporary directories defined in the system. Newly downloaded payloads will
589 be stored in this directory in a hashed structure, see `payload_locations`
590 download_lock_timeout (int): How many seconds to wait for a write lock when
591 concurrently downloading the same payload between different processes.
592 If locking fails the payload will be downloaded to a temporary file
593 separately for each process.
594 usable_globaltag_states (set(str)): Names of globaltag states accepted for
595 processing. This can be changed to make sure that only fully published
596 globaltags are used or to enable running on an open tag. It is not possible
597 to allow usage of 'INVALID' tags, those will always be rejected.
598 connection_timeout (int): timeout in seconds before connection should be
599 aborted. 0 sets the timeout to the default (300s)
600 stalled_timeout (int): timeout in seconds before a download should be
601 aborted if the speed stays below 10 KB/s, 0 disables this timeout
602 max_retries (int): maximum amount of retries if the server responded with
603 an HTTP response of 500 or more. 0 disables retrying
604 backoff_factor (int): backoff factor for retries in seconds. Retries are
605 performed using something similar to binary backoff: For retry :math:`n`
606 and a ``backoff_factor`` :math:`f` we wait for a random time chosen
607 uniformly from the interval :math:`[1, (2^{n} - 1) \times f]` in
612Set a callback function to be called just before processing.
614This callback can be used to further customize the globaltags to be used during
615processing. It will be called after the input files have been opened and checked
616with three keyword arguments:
619 The globaltags determined from either the input files or, if no input files
620 are present, the default globaltags
623 The globaltags provided by the user
626 If there are not input files (e.g. generating events) this argument is None.
627 Otherwise it is a list of all the ``FileMetaData`` instances from all input files.
628 This list can be empty if there is no metadata associated with the input files.
630From this information the callback function should then compose the final list
631of globaltags to be used for processing and return this list. If ``None`` is
632returned the default behavior is applied as if there were no callback function.
633If anything else is returned the processing is aborted.
635If no callback function is specified the default behavior is equivalent to ::
637 def callback(base_tags, user_tags, metadata):
639 basf2.B2FATAL("No baseline globaltags available. Please use override")
641 return user_tags + base_tags
643If `override_enabled` is ``True`` then the callback function will not be called.
646 If a callback is set it is responsible to select the correct list of globaltags
647 and also make sure that all files are compatible. No further checks will be
648 done by the framework but any list of globaltags which is returned will be used
651 If the list of ``base_tags`` is empty that usually means that the input files
652 had different globaltag settings but it is the responsibility of the callback
653 to then verify if the list of globaltags is usable or not.
655 If the callback function determines that no working set of globaltags can be
656 determined then it should abort processing using a FATAL error or an exception
bool m_overrideEnabled
is the globaltag override enabled?
CppOrPyList m_globalTags
the list with all user globaltags
std::string m_defaultRemoteMetadataProviderServer
default server URL for the remote metadata provider
void prependGlobalTag(const std::string &globalTag)
prepend a globaltag
void appendGlobalTag(const std::string &globalTag)
Append a globaltag.
Configuration()
Initialize default values.
void ensureEditable() const
Check whether the configuration object can be edited or if the database has been initialized already.
void setGlobaltagCallbackPy(const boost::python::object &obj)
Set a callback function from python which will be called when processing starts and should return the...
void disableGlobalTagReplay()
Disable global tag replay.
boost::python::list getGlobalTagsPy()
Get the list of user globaltags as python version.
void setMetadataProvidersPy(const boost::python::list &list)
Set the list of metadata providers in python.
boost::python::list getTestingPayloadLocationsPy()
Get the list of text files containing test payloads in python.
std::string m_defaultLocalMetadataProviderPath
default local path for the local metadata provider
std::vector< std::string > getFinalListOfTags()
Get the final list of globaltags to be used for processing.
boost::python::tuple getDefaultGlobalTagsPy() const
Get the tuple of default globaltags as python version.
static void fillFromEnv(T &target, const std::string &envName, const std::string &defaultValue)
Fill a target object from a list of environment variables.
std::vector< FileMetaData > m_inputMetadata
the file metadata of all input files if globaltag replay is requested by input module
static Configuration & getInstance()
Get a reference to the instance which will be used when the Database is initialized.
void setPayloadLocationsPy(const boost::python::list &list)
Set the list of payload locations in python.
CppOrPyList m_metadataProviders
the list with all the metadata providers
void setInputMetadata(const std::vector< FileMetaData > &inputMetadata)
To be called by input modules with the list of all input FileMetaData.
boost::python::list getMetadataProvidersPy()
Get the list of metadata providers in python.
void overrideGlobalTags()
Enable globaltag override: If this is called once than overrideEnabled() will return true and getFina...
std::vector< std::string > getBaseTags() const
Get the base globaltags to be used in addition to user globaltags.
void setTestingPayloadLocationsPy(const boost::python::list &list)
Set the list of text files containing test payloads in python.
static void exposePythonAPI()
expose this class to python
std::optional< std::vector< std::string > > m_inputGlobaltags
the list of globaltags from all the input files to be used in addition to the user globaltags
void prependTestingPayloadLocation(const std::string &filename)
Prepend a local text file with testing payloads to the list.
boost::python::list getPayloadLocationsPy()
Get the list og payload locations in python.
CppOrPyList m_payloadLocations
the list with all the payload locations
std::vector< std::string > getDefaultGlobalTags() const
Get the std::vector of default globaltags.
void overrideGlobalTagsPy(const boost::python::list &globalTags)
Enable globaltag override and set the list of user globaltags in one go.
void reset()
Reset to default values.
std::string getDefaultRemoteMetadataProviderServer()
Get the default server URL for the remote metadata provider.
void setGlobalTagsPy(const boost::python::list &globalTags)
Set the list of globaltags from python.
bool overrideEnabled() const
Check if override is enabled by previous calls to overrideGlobalTags()
std::optional< boost::python::object > m_callback
the callback function to determine the final final list of globaltags
void appendTestingPayloadLocation(const std::string &filename)
Add a local text file with testing payloads.
bool m_databaseInitialized
bool indicating whether the database has been initialized, in which case any changes to the configura...
void prepend(const std::string &element)
Prepend an element to whatever representation we currently have.
void shallowCopy(const boost::python::object &source)
shallow copy all elements of the source object into the python representation.
void append(const std::string &element)
Append an element to whatever representation we currently have.
boost::python::list & ensurePy()
Return the python list version.
std::vector< std::string > & ensureCpp()
Return the C++ vector version.
std::variant< std::vector< std::string >, boost::python::list > m_value
Store either a std::vector or a python list of strings.
static Downloader & getDefaultInstance()
Return the default instance.
static std::string get(const std::string &name, const std::string &fallback="")
Get the value of an environment variable or the given fallback value if the variable is not set.
static bool isSet(const std::string &name)
Check if a value is set in the database.
boost::python::object createROOTObjectPyCopy(const T &instance)
Create a python wrapped copy from a class instance which has a ROOT dictionary.
static Database & Instance()
Instance of a singleton Database.
static std::vector< std::string > getOrCreateList(const std::string &name, const std::string &fallback, const std::string &separators=" \t\n\r")
Get a list of values from an environment variable or the given fallback string if the variable is not...
static void reset(bool keepConfig=false)
Reset the database instance.
boost::python::object convertToPythonObject(const Scalar &value)
------------— From C++ TO Python Converter ---------------------—
bool iteratePythonObject(const boost::python::object &pyObject, Functor function)
Helper function to loop over a python object that implements the iterator concept and call a functor ...
Scalar convertPythonObject(const boost::python::object &pyObject, Scalar)
Convert from Python to given type.
static std::string name()
type name.
Helper struct for the C++17 std::visit overload pattern to allow simple use of variants.