 |
Belle II Software
release-05-01-25
|
11 #include <framework/database/Configuration.h>
12 #include <framework/logging/Logger.h>
13 #include <framework/dataobjects/FileMetaData.h>
14 #include <framework/database/Downloader.h>
15 #include <framework/database/Database.h>
16 #include <framework/utilities/Utils.h>
17 #include <boost/python.hpp>
18 #include <framework/core/PyObjConvUtils.h>
19 #include <framework/core/PyObjROOTUtils.h>
20 #include <boost/algorithm/string.hpp>
26 #define CURRENT_DEFAULT_TAG "release-05-01-09"
28 namespace py = boost::python;
36 std::vector<std::string> extractStringList(
const py::object& obj)
38 std::vector<std::string> result;
40 py::object str(py::handle<>(PyObject_Str(item.ptr())));
41 py::extract<std::string> extract(str);
42 result.emplace_back(extract());
50 namespace Belle2::Conditions {
55 boost::python::list tmp;
56 for (
const auto& e : std::get<0>(
m_value)) { tmp.append(e); }
57 m_value.emplace<boost::python::list>(std::move(tmp));
66 std::vector<std::string> tmp = extractStringList(std::get<1>(
m_value));
67 m_value.emplace<std::vector<std::string>>(std::move(tmp));
75 [&element](std::vector<std::string>& list) {list.emplace_back(element);},
76 [&element](boost::python::list & list) {list.append(element);}
83 [&element](std::vector<std::string>& list) {list.emplace(list.begin(), element);},
84 [&element](boost::python::list & list) {list.insert(0, element);}
90 ensurePy().slice(boost::python::_, boost::python::_) = source;
131 fillFromEnv(list,
"BELLE2_CONDB_GLOBALTAG", CURRENT_DEFAULT_TAG);
132 return py::tuple(list);
143 if (inputMetadata.empty())
return;
145 std::optional<std::string> inputGlobaltags;
146 for (
const auto& metadata : inputMetadata) {
147 if (!inputGlobaltags) {
148 inputGlobaltags = metadata.getDatabaseGlobalTag();
150 if (inputGlobaltags != metadata.getDatabaseGlobalTag()) {
151 B2WARNING(
"Input files metadata contain incompatible globaltag settings, globaltag replay not possible");
158 if (inputGlobaltags and inputGlobaltags->empty()) {
159 B2WARNING(
"Input files metadata all have empty globaltag setting, globaltag replay not possible");
175 std::optional<std::string> youngest;
176 for (
const auto& metadata : inputMetadata) {
178 const std::string& release = metadata.getRelease();
179 if (release.substr(0, 8) ==
"release-" and
180 release.compare(8, 2,
"04", 2) >= 0)
183 if (!youngest or * youngest > metadata.getDate()) {
184 youngest = metadata.getDate();
187 if (youngest and youngest->compare(
"2019-12-31") < 0) {
188 B2DEBUG(30,
"Enabling legacy IP information globaltag in tag replay");
205 B2INFO(
"Global tag override is in effect: input globaltags and default globaltag will be ignored");
216 for (
const auto& tag : baseList) baseListPy.append(tag);
217 arguments[
"base_tags"] = baseListPy;
223 arguments[
"metadata"] = py::object();
226 py::list metaDataList;
228 arguments[
"metadata"] = metaDataList;
231 py::object retval = (*m_callback)(*py::tuple(), **arguments);
234 if (retval != py::object()) {
235 return extractStringList(retval);
240 if (baseList.empty()) {
242 B2FATAL(R
"(No baseline globaltags available.
243 The input files you selected don't have compatible globaltags or an empty
244 globaltag setting. As such globaltag configuration cannot be determined
247 If you really sure that it is a good idea to process these files together
248 you have to manually override the list of globaltags:
250 >>> basf2.conditions.override_globaltags()
253 B2FATAL(R
"(No default globaltags available.
254 There is no default globaltag available for processing. This usually means
255 you set the environment variable BELLE2_CONDB_GLOBALTAG to an empty value.
257 As this is unlikely to work for even the most basic functionality this is not
258 directly supported anymore. If you really want to disable any access to the
259 conditions database please configure this explicitly
261 >>> basf2.conditions.metadata_providers = []
262 >>> basf2.conditions.override_globaltags([])
268 for (
const auto& tag : baseList) { finalList.emplace_back(tag); }
274 boost::python::dict expertSettings(
const boost::python::tuple& args, boost::python::dict kwargs)
276 if (py::len(args) != 1) {
278 PyErr_SetString(PyExc_TypeError, (
"expert_settings() takes one positional argument but " +
279 std::to_string(len(args)) +
" were given").c_str());
280 py::throw_error_already_set();
282 Configuration&
self = py::extract<Configuration&>(args[0]);
289 auto checkValue = [&kwargs, &result](
const std::string & name,
auto setter,
auto getter) {
290 using value_type = decltype(getter());
291 if (kwargs.has_key(name)) {
293 py::object
object = kwargs[name];
296 }
catch (std::runtime_error&) {
297 std::stringstream error;
299 PyErr_SetString(PyExc_TypeError, error.str().c_str());
300 py::throw_error_already_set();
304 py::delitem(kwargs, py::object(name));
311 checkValue(
"save_payloads",
312 [&
self](
const std::string & path) {
self.setNewPayloadLocation(path);},
313 [&
self]() {
return self.getNewPayloadLocation();});
314 checkValue(
"download_cache_location",
315 [&
self](
const std::string & path) {
self.setDownloadCacheDirectory(path);},
316 [&
self]() {
return self.getDownloadCacheDirectory();});
317 checkValue(
"download_lock_timeout",
318 [&
self](
size_t timeout) {
self.setDownloadLockTimeout(timeout);},
319 [&
self]() {
return self.getDownloadLockTimeout();});
320 checkValue(
"usable_globaltag_states",
321 [&
self](
const auto & states) {
self.setUsableTagStates(states); },
322 [&
self]() {
return self.getUsableTagStates(); });
323 checkValue(
"connection_timeout",
324 [&downloader](
unsigned int timeout) {downloader.setConnectionTimeout(timeout);},
325 [&downloader]() {
return downloader.getConnectionTimeout();});
326 checkValue(
"stalled_timeout",
327 [&downloader](
unsigned int timeout) {downloader.setStalledTimeout(timeout);},
328 [&downloader]() {
return downloader.getStalledTimeout();});
329 checkValue(
"max_retries",
330 [&downloader](
unsigned int retries) {downloader.setMaxRetries(retries);},
331 [&downloader]() {
return downloader.getMaxRetries();});
332 checkValue(
"backoff_factor",
333 [&downloader](
unsigned int factor) { downloader.setBackoffFactor(factor);},
334 [&downloader]() {
return downloader.getBackoffFactor();});
336 if (py::len(kwargs) > 0) {
337 std::string message =
"Unrecognized keyword arguments: ";
338 auto keys = kwargs.keys();
339 for (
int i = 0; i < len(keys); ++i) {
340 if (i > 0) message +=
", ";
341 message += py::extract<std::string>(keys[i]);
343 PyErr_SetString(PyExc_TypeError, message.c_str());
344 py::throw_error_already_set();
359 py::docstring_options options(
true,
false,
false);
363 py::object expert = raw_function(expertSettings);
364 py::class_<Configuration>(
"ConditionsConfiguration", R
"DOC(
365 This class contains all configurations for the conditions database service
367 * which globaltags to use
368 * where to look for payload information
369 * where to find the actual payload files
370 * which temporary testing payloads to use
372 But for most users the only thing they should need to care about is to set the
373 list of additional `globaltags` to use.
376 Indicator whether or not the override of globaltags is enabled. If true then
377 globaltags present in input files will be ignored and only the ones given in
378 `globaltags` will be considered.
382 Reset the conditions database configuration to its original state.
385 A tuple containing the default globaltags to be used if events are generated without an input file.
388 List of globaltags to be used. These globaltags will be the ones with highest
389 priority but by default the globaltags used to create the input files or the
390 default globaltag will also be used.
392 The priority of the globaltags in this list is highest first. So the first in
393 the list will be checked first and all other globaltags will only be checked for
394 payloads not found so far.
397 By default this list contains the globaltags to be used **in addition** to
398 the ones from the input file or the default one if no input file is present.
399 If this is not desirable you need to call `override_globaltags()` to disable
400 any addition or modification of this list.
404 Append a globaltag to the end of the `globaltags` list. That means it will be
405 the lowest priority of all tags in the list.
409 Add a globaltag to the beginning of the `globaltags` list. That means it will be
410 the highest priority of all tags in the list.
412 .def("override_globaltags", overrideGTFlag)
413 .def(
"override_globaltags", overrideGTList, py::args(
"globaltags"), R
"DOC(override_globaltags(list=None)
415 Enable globaltag override. This disables all modification of the globaltag list at the beginning of processing:
417 * the default globaltag or the input file globaltags will be ignored.
418 * any callback set with `set_globaltag_callback` will be ignored.
419 * the list of `globaltags` will be used exactly as it is.
422 list (list(str) or None) if given this list will replace the current content of `globaltags`
425 it's still possible to modify `globaltags` after this call.
429 Disable global tag replay and revert to the old behavior that the default
430 globaltag will be used if no other globaltags are specified.
432 This is a shortcut to just calling
434 >>> conditions.override_globaltags()
435 >>> conditions.globaltags += list(conditions.default_globaltags)
440 Append a text file containing local test payloads to the end of the list of
441 `testing_payloads`. This will mean they will have lower priority than payloads
442 in previously defined text files but still higher priority than globaltags.
445 filename (str): file containing a local definition of payloads and their
446 intervals of validity for testing
449 This functionality is strictly for testing purposes. Using local payloads
450 leads to results which cannot be reproduced by anyone else and thus cannot
455 Insert a text file containing local test payloads in the beginning of the list
456 of `testing_payloads`. This will mean they will have higher priority than payloads in
457 previously defined text files as well as higher priority than globaltags.
460 filename (str): file containing a local definition of payloads and their
461 intervals of validity for testing
464 This functionality is strictly for testing purposes. Using local payloads
465 leads to results which cannot be reproduced by anyone else and thus cannot
469 List of text files to look for local testing payloads. Each entry should be a
470 text file containing local payloads and their intervals of validity to be used
473 Payloads found in these files and valid for the current run will have a higher
474 priority than any of the `globaltags`. If a valid payload is present in multiple
475 files the first one in the list will have higher priority.
478 This functionality is strictly for testing purposes. Using local payloads
479 leads to results which cannot be reproduced by anyone else and thus cannot
483 List of metadata providers to use when looking for payload metadata. There are currently two supported providers:
485 1. Central metadata provider to look for payloads in the central conditions database.
486 This provider is used for any entry in this list which starts with ``http(s)://``.
487 The URL should point to the top level of the REST api endpoints on the server
489 2. Local metadata provider to look for payloads in a local SQLite snapshot taken
490 from the central server. This provider will be assumed for any entry in this
491 list not starting with a protocol specifier or if the protocol is given as ``file://``
493 This list should rarely need to be changed. The only exception is for users who
494 want to be able to use the software without internet connection after they
495 downloaded a snapshot of the necessary globaltags with ``b2conditionsdb download``
496 to point to this location.
499 List of payload locations to search for payloads which have been found by any of
500 the configured `metadata_providers`. This can be a local directory or a
501 ``http(s)://`` url pointing to the payload directory on a server.
503 For remote locations starting with ``http(s)://`` we assume that the layout of
504 the payloads on the server is the same as on the main payload server:
505 The combination of given location and the relative url in the payload metadata
506 field ``payloadUrl`` should point to the correct payload on the server.
508 For local directories, two layouts are supported and will be auto detected:
511 All payloads are in the same directory without any substructure with the name
512 ``dbstore_{name}_rev_{revision}.root``
514 All payloads are stored in subdirectories in the form ``AB/{name}_r{revision}.root``
515 where ``A`` and ``B`` are the first two characters of the md5 checksum of the
519 Given ``payload_locations = ["payload_dir/", "http://server.com/payloads"]``
520 the framework would look for a payload with name ``BeamParameters`` in revision
521 ``45`` (and checksum ``a34ce5...``) in the following places
524 1. ``payload_dir/a3/BeamParameters_r45.root``
525 2. ``payload_dir/dbstore_BeamParameters_rev_45.root``
526 3. ``http://server.com/payloads/dbstore/BeamParameters/dbstore_BeamParameters_rev_45.root``
527 given the usual pattern of the ``payloadUrl`` metadata. But this could be
528 changed on the central servers so mirrors should not depend on this convention
529 but copy the actual structure of the central server.
531 If the payload cannot be found in any of the given locations the framework will
532 always attempt to download it directly from the central server and put it in a
533 local cache directory.
535 .def("expert_settings", expert, R
"DOC(expert_settings(**kwargs)
537 Set some additional settings for the conditions database.
539 You can supply any combination of keyword-only arguments defined below. The
540 function will return a dictionary containing all current settings.
542 >>> conditions.expert_settings(connection_timeout=5, max_retries=1)
543 {'save_payloads': 'localdb/database.txt',
544 'download_cache_location': '',
545 'download_lock_timeout': 120,
546 'usable_globaltag_states': {'PUBLISHED', 'RUNNING', 'TESTING', 'VALIDATED'},
547 'connection_timeout': 5,
548 'stalled_timeout': 60,
553 Modification of these parameters should not be needed, in rare
554 circumstances this could be used to optimize access for many jobs at once
555 but should only be set by experts.
558 save_payloads (str): Where to store new payloads created during processing.
559 This should be a filename to contain the payload information and the payload
560 files will be placed in the same directory as the file.
561 download_cache_location (str): Where to store payloads which have been downloaded
562 from the central server. This could be a user defined directory, otherwise
563 empty string defaults to ``$TMPDIR/basf2-conditions`` where ``$TMPDIR`` is the
564 temporary directories defined in the system. Newly downloaded payloads will
565 be stored in this directory in a hashed structure, see `payload_locations`
566 download_lock_timeout (int): How many seconds to wait for a write lock when
567 concurrently downloading the same payload between different processes.
568 If locking fails the payload will be downloaded to a temporary file
569 separately for each process.
570 usable_globaltag_states (set(str)): Names of globaltag states accepted for
571 processing. This can be changed to make sure that only fully published
572 globaltags are used or to enable running on an open tag. It is not possible
573 to allow usage of 'INVALID' tags, those will always be rejected.
574 connection_timeout (int): timeout in seconds before connection should be
575 aborted. 0 sets the timeout to the default (300s)
576 stalled_timeout (int): timeout in seconds before a download should be
577 aborted if the speed stays below 10 KB/s, 0 disables this timeout
578 max_retries (int): maximum amount of retries if the server responded with
579 an HTTP response of 500 or more. 0 disables retrying
580 backoff_factor (int): backoff factor for retries in seconds. Retries are
581 performed using something similar to binary backoff: For retry :math:`n`
582 and a ``backoff_factor`` :math:`f` we wait for a random time chosen
583 uniformely from the interval :math:`[1, (2^{n} - 1) \times f]` in
588 Set a callback function to be called just before processing.
590 This callback can be used to further customize the globaltags to be used during
591 processing. It will be called after the input files have been opened and checked
592 with three keyword arguments:
595 The globaltags determined from either the input files or, if no input files
596 are present, the default globaltags
599 The globaltags provided by the user
602 If there are not input files (e.g. generating events) this argument is None.
603 Otherwise it is a list of all the ``FileMetaData`` instances from all input files.
604 This list can be empty if there is no metadata associated with the input files.
606 From this information the callback function should then compose the final list
607 of globaltags to be used for processing and return this list. If ``None`` is
608 returned the default behavior is applied as if there were no callback function.
609 If anything else is returned the processing is aborted.
611 If no callback function is specified the default behavior is equivalent to ::
613 def callback(base_tags, user_tags, metadata):
615 basf2.B2FATAL("No baseline globaltags available. Please use override")
617 return user_tags + base_tags
619 If `override_enabled` is ``True`` then the callback function will not be called.
622 If a callback is set it is responsible to select the correct list of globaltags
623 and also make sure that all files are compatible. No further checks will be
624 done by the framework but any list of globaltags which is returned will be used
627 If the list of ``base_tags`` is empty that usually means that the input files
628 had different globaltag settings but it is the responsibility of the callback
629 to then verify if the list of globaltags is usable or not.
631 If the callback function determines that no working set of globaltags can be
632 determined then it should abort processing using a FATAL error or an exception
void prependGlobalTag(const std::string &globalTag)
preprend a globaltag
bool m_databaseInitialized
bool indicating whether the database has been initialized, in which case any changes to the configura...
boost::python::list getPayloadLocationsPy()
Get the list og payload locations in python.
static std::vector< std::string > getOrCreateList(const std::string &name, const std::string &fallback, const std::string &separators=" \t\n\r")
Get a list of values from an environment variable or the given fallback string if the variable is not...
bool overrideEnabled() const
Check if override is enabled by previous calls to overrideGlobalTags()
void prepend(const std::string &element)
Prepend an element to whatever representation we currently have.
boost::python::object convertToPythonObject(const Scalar &value)
------------— From C++ TO Python Converter ---------------------—
boost::python::list getGlobalTagsPy()
Get the list of user globaltags as python version.
static void fillFromEnv(T &target, const std::string &envName, const std::string &defaultValue)
Fill a target object from a list of environment variables.
void overrideGlobalTags()
Enable globaltag override: If this is called once than overrideEnabled() will return true and getFina...
std::variant< std::vector< std::string >, boost::python::list > m_value
Store either a std::vector or a python list of strings.
bool m_overrideEnabled
is the globaltag override enabled?
VisitOverload(Ts...) -> VisitOverload< Ts... >
Function for the C++17 std::visit overload pattern to allow simple use of variants.
void setGlobaltagCallbackPy(const boost::python::object &obj)
Set a callback function from python which will be called when processing starts and should return the...
CppOrPyList m_metadataProviders
the list with all the metadata providers
void append(const std::string &element)
Append an element to whatever representation we currently have.
boost::python::object createROOTObjectPyCopy(const T &instance)
Create a python wrapped copy from a class instance which has a ROOT dictionary.
std::vector< FileMetaData > m_inputMetadata
the file metadata of all input files if globaltag replay is requested by input module
std::vector< std::string > & ensureCpp()
Return the C++ vector version.
void setMetadataProvidersPy(const boost::python::list &list)
Set the list of metadata providers in python.
std::optional< std::vector< std::string > > m_inputGlobaltags
the list of globaltags from all the input files to be used in addition to the user globaltags
static void exposePythonAPI()
expose this class to python
boost::python::list getTestingPayloadLocationsPy()
Get the list of text files containing test payloads in python.
static std::string get(const std::string &name, const std::string &fallback="")
Get the value of an environment variable or the given fallback value if the variable is not set.
boost::python::list getMetadataProvidersPy()
Get the list of metadata providers in python.
static void reset(bool keepConfig=false)
Reset the database instance.
std::vector< std::string > getBaseTags() const
Get the base globaltags to be used in addition to user globaltags.
std::optional< boost::python::object > m_callback
the callback function to determine the final final list of globaltags
CppOrPyList m_payloadLocations
the list with all the payload locations
static Configuration & getInstance()
Get a reference to the instance which will be used when the Database is initialized.
std::vector< std::string > getFinalListOfTags()
Get the final list of globaltags to be used for processing.
static Downloader & getDefaultInstance()
Return the default instance.
bool iteratePythonObject(const boost::python::object &pyObject, Functor function)
Helper function to loop over a python object that implements the iterator concept and call a functor ...
static bool isSet(const std::string &name)
Check if a value is set in the database.
void overrideGlobalTagsPy(const boost::python::list &globalTags)
Enable globaltag override and set the list of user globaltags in one go.
std::vector< std::string > getDefaultGlobalTags() const
Get the std::vector of default globaltags.
void ensureEditable() const
Check whether the configuration object can be edited or if the database has been initialized already.
void appendGlobalTag(const std::string &globalTag)
Append a globaltag.
static std::string name()
type name.
static Database & Instance()
Instance of a singleton Database.
void setInputMetadata(const std::vector< FileMetaData > &inputMetadata)
To be called by input modules with the list of all input FileMetaData.
void appendTestingPayloadLocation(const std::string &filename)
Add a local text file with testing payloads.
boost::python::list & ensurePy()
Return the python list version.
Configuration()
Initialize default values.
void setTestingPayloadLocationsPy(const boost::python::list &list)
Set the list of text files containing test payloads in python.
void setGlobalTagsPy(const boost::python::list &globalTags)
Set the list of globaltags from python.
void setPayloadLocationsPy(const boost::python::list &list)
Set the list of payload locations in python.
void prependTestingPayloadLocation(const std::string &filename)
Prepend a local text file with testing payloads to the list.
void disableGlobalTagReplay()
Disable global tag replay.
CppOrPyList m_globalTags
the list with all user globaltags
void reset()
Reset to default values.
void shallowCopy(const boost::python::object &source)
shallow copy all elements of the source object into the python representation.
boost::python::tuple getDefaultGlobalTagsPy() const
Get the tuple of default globaltags as python version.
Scalar convertPythonObject(const boost::python::object &pyObject, Scalar)
Convert from Python to given type.