Belle II Software development
Database.cc
1/**************************************************************************
2 * basf2 (Belle II Analysis Software Framework) *
3 * Author: The Belle II Collaboration *
4 * *
5 * See git log for contributors and copyright holders. *
6 * This file is licensed under LGPL-3.0, see LICENSE.md. *
7 **************************************************************************/
8
9#include <boost/python/def.hpp>
10#include <boost/python/overloads.hpp>
11#include <boost/python/docstring_options.hpp>
12#include <boost/python/list.hpp>
13#include <boost/python/dict.hpp>
14#include <boost/python/extract.hpp>
15#include <boost/python/return_value_policy.hpp>
16#include <boost/python/manage_new_object.hpp>
17#include <boost/python/raw_function.hpp>
18#include <boost/algorithm/string.hpp>
19
20#include <framework/database/Database.h>
21
22#include <framework/dataobjects/EventMetaData.h>
23#include <framework/logging/Logger.h>
24#include <framework/database/DBStore.h>
25
26#include <framework/database/PayloadProvider.h>
27#include <framework/database/MetadataProvider.h>
28#include <framework/database/LocalMetadataProvider.h>
29#include <framework/database/CentralMetadataProvider.h>
30#include <framework/database/Configuration.h>
31
32#include <algorithm>
33#include <cstdlib>
34
35namespace Belle2 {
40
42 {
43 static Database instance;
44 return instance;
45 }
46
47 Database::~Database() = default;
48
49 void Database::reset(bool keepConfig)
50 {
52 conf.setInitialized(false);
57 if (not keepConfig)
58 conf.reset();
59 }
60
62 {
63 // make sure we reread testing text files in case they got updated
64 for (auto& testing : m_testingPayloads) {
65 testing.reset();
66 }
67 // and return a downloader session guard for the downloader we use
69 }
70
71 std::pair<TObject*, IntervalOfValidity> Database::getData(const EventMetaData& event, const std::string& name)
72 {
73 DBStoreEntry entry(DBStoreEntry::c_Object, name, TObject::Class(), false, true);
74 std::vector<DBQuery> query{DBQuery{name, true}};
75 getData(event, query);
76 entry.updatePayload(query[0].revision, query[0].iov, query[0].filename, query[0].checksum, query[0].globaltag, event);
77 return std::make_pair(entry.releaseObject(), query[0].iov);
78 }
79
80 bool Database::getData(const EventMetaData& event, std::vector<DBQuery>& query)
81 {
82 // initialize lazily ...
84 // So first go over the requested payloads once, reset the info and check for any
85 // testing payloads we might want to use
86 const size_t testingPayloads = std::count_if(query.begin(), query.end(), [this, &event](auto & payload) {
87 // make sure the queries are "reset" to invalid revision and no filename before we start looking
88 payload.filename = "";
89 payload.revision = 0;
90 // and now look in all testing payload providers if any.
91 for (auto& tmp : m_testingPayloads) {
92 if (tmp.get(event, payload)) return true;
93 }
94 return false;
95 });
96 // if we already found all just return here
97 if (testingPayloads == query.size()) return true;
98 // nooow, lets look for proper payloads;
99 try {
100 m_metadataProvider->getPayloads(event.getExperiment(), event.getRun(), query);
101 } catch (std::exception&) {
102 // something went wrong with the metadata update ... so let's try next provider
103 B2WARNING("Conditions data: Problem with payload metadata provider, trying to fall back to next provider..."
104 << LogVar("provider", m_currentProvider));
105 nextMetadataProvider();
106 return getData(event, query);
107 }
108 // and if we could find the metadata lets also locate the payloads ...
109 const size_t payloadsLocated = std::count_if(query.begin(), query.end(), [this](auto & payload) {
110 // make sure we don't overwrite local payloads or otherwise already valid filenames;
111 if (!payload.filename.empty()) return true;
112 // but don't check for payloads we could not find. But this is only a
113 // problem if they are required so report success for not required
114 // payloads
115 if (payload.revision == 0) return not payload.required;
116 // and locate the payload.
117 if (not m_payloadProvider->find(payload)) {
118 // if that fails lets let the user know: Even for optional payloads, if
119 // we know the metadata but cannot find the file something is fishy and
120 // should be reported.
121 auto loglevel = payload.required ? LogConfig::c_Error : LogConfig::c_Warning;
122 B2LOG(loglevel, 0, "Conditions data: Could not find file for payload"
123 << LogVar("name", payload.name) << LogVar("revision", payload.revision)
124 << LogVar("checksum", payload.checksum) << LogVar("globaltag", payload.globaltag));
125 return not payload.required;
126 }
127 return true;
128 });
129 // did we find all payloads?
130 return payloadsLocated == query.size();
131 }
132
133 bool Database::storeData(std::list<DBImportQuery>& query)
134 {
135 return std::all_of(query.begin(), query.end(), [this](const auto & import) {
136 return storeData(import.name, import.object, import.iov);
137 });
138 }
139
140 bool Database::storeData(const std::string& name, TObject* obj, const IntervalOfValidity& iov)
141 {
143 auto result = m_payloadCreation->storeData(name, obj, iov);
144 // we added payloads, make sure we reread testing files on next try
145 if (result) {
146 for (auto& testing : m_testingPayloads) {
147 testing.reset();
148 }
149 }
150 return result;
151 }
152
154 {
155 return boost::algorithm::join(m_globalTags, ",");
156 }
157
159 {
160 if (m_metadataConfigurations.empty()) {
161 B2FATAL("Conditions data: No more metadata providers available");
162 }
164 m_metadataConfigurations.pop_back();
165 bool remote{false};
166 if (auto pos = m_currentProvider.find("://"); pos != std::string::npos) {
167 // found a protocol: if file remove, otherwise keep as is and set as remote ...
168 auto protocol = m_currentProvider.substr(0, pos);
169 boost::algorithm::to_lower(protocol);
170 if (protocol == "file") {
171 m_currentProvider = m_currentProvider.substr(pos + 3);
172 } else if (protocol == "http" or protocol == "https") {
173 remote = true;
174 } else {
175 B2FATAL("Conditions data: Unknown metadata protocol, only supported protocols for payload metadata are file, http, https"
176 << LogVar("protocol", protocol));
177 }
178 }
179 try {
180 if (remote) {
181 m_metadataProvider = std::make_unique<Conditions::CentralMetadataProvider>(m_currentProvider, m_usableTagStates);
182 } else {
183 m_metadataProvider = std::make_unique<Conditions::LocalMetadataProvider>(m_currentProvider, m_usableTagStates);
184 }
185 } catch (std::exception& e) {
186 B2WARNING("Conditions data: Metadata provider not usable, trying next one ..."
187 << LogVar("provider", m_currentProvider) << LogVar("error", e.what()));
188 return nextMetadataProvider();
189 }
190 // and check the tags are usable: if not, fallback to the next metadata provider
191 if (!m_metadataProvider->setTags(m_globalTags)) {
192 B2WARNING("Conditions data: One (or more) globaltag is not available in the current metadata proivder, trying next one ...");
193 return nextMetadataProvider();
194 }
195 }
196
198 {
199 if (m_configState >= target) return;
201
202 if (m_configState == c_PreInit) {
203 // first step: freeze the configuration object and determine the list of globaltags
204 // this calculates if tag replay is possible and will create an error otherwise but
205 // it will not do anything else than setting the final list of globaltags
206 conf.setInitialized(true);
207 m_globalTags = conf.getFinalListOfTags();
208 // trim the globaltag names for removing leading/trailing whitespaces, since they
209 // are a potential source of FATAL errors
210 std::transform(m_globalTags.begin(), m_globalTags.end(), m_globalTags.begin(),
211 [](const auto & tag) {
212 return boost::algorithm::trim_copy(tag);
213 });
214 // and remove duplicates, there's no need to look in the same gt multiple times
215 std::set<std::string> seen;
216 m_globalTags.erase(std::remove_if(m_globalTags.begin(), m_globalTags.end(),
217 [&seen](const auto & tag) {
218 return not seen.insert(tag).second;
219 }), m_globalTags.end());
220 // and also obtain usable tag states and metadata providers
221 m_usableTagStates = conf.getUsableTagStates();
222 m_metadataConfigurations = conf.getMetadataProviders();
223 // reverse because we want to pop out elements when used
224 std::reverse(m_metadataConfigurations.begin(), m_metadataConfigurations.end());
227 }
228 // do we want to stop early?
229 if (m_configState >= target) return;
231 // setup the first working provider;
232 if (m_metadataConfigurations.empty()) {
233 m_metadataProvider = std::make_unique<Conditions::NullMetadataProvider>();
234 } else {
236 }
237 // we will actually use the globaltags so print them now
238 if (!m_globalTags.empty()) {
239 // Globaltags are usable so print out the final list we're gonna use
240 if (m_globalTags.size() == 1) {
241 B2INFO("Conditions data: configured globaltag is " << m_globalTags[0]);
242 } else {
243 B2INFO("Conditions data: configured globaltags (highest priority first) are " << boost::algorithm::join(m_globalTags, ", "));
244 }
245 }
246 // Configure payload location/download
247 m_payloadProvider = std::make_unique<Conditions::PayloadProvider>(
248 conf.getPayloadLocations(),
249 conf.getDownloadCacheDirectory(),
250 conf.getDownloadLockTimeout()
251 );
252 // Also we need to be able to create payloads ...
253 m_payloadCreation = std::make_unique<Conditions::TestingPayloadStorage>(conf.getNewPayloadLocation());
254 // And maaaybe we want to use testing payloads
255 m_testingPayloads.clear();
256 for (const auto& path : conf.getTestingPayloadLocations()) {
257 B2INFO("Conditions data: configured to use testing payloads" << LogVar("location", path));
258 m_testingPayloads.emplace_back(path);
259 }
260 // If so, warn again ... because
261 if (not m_testingPayloads.empty()) {
262 B2WARNING(R"(Conditions data: configured to look for temporary testing payloads from one or more local folders.
263
264 This will lead to non-reproducible results and is strictly only for testing purposes.
265 It is NOT ALLOWED for official productions or data analysis and any results obtained like this WILL NOT BE PUBLISHED.
266 )");
267 }
269 }
270 }
271
273 {
274 // To avoid confusion between std::arg and boost::python::arg we want a shorthand namespace as well
275 namespace py = boost::python;
276
277 // Make sure the default instance is created
279
280 // Don't show c++ signature in python doc to keep it simple
281 py::docstring_options options(true, true, false);
282
283 // Expose our shiny configuration object
285 }
286
287}
static Configuration & getInstance()
Get a reference to the instance which will be used when the Database is initialized.
static void exposePythonAPI()
expose this class to python
ScopeGuard ensureSession()
Make sure there's an active session and return a ScopeGuard object that closes the session on destruc...
Definition Downloader.h:43
static Downloader & getDefaultInstance()
Return the default instance.
Class to hold one entry from the ConditionsDB in the DBStore.
@ c_Object
A ROOT file containing a object with the name of the DBStoreEntry.
TObject * releaseObject()
Return the pointer to the current object and release ownership: The caller is responsible to clean up...
std::string m_currentProvider
Name of the currently used metadata provider.
Definition Database.h:232
std::unique_ptr< Conditions::MetadataProvider > m_metadataProvider
Currently active metadata provider.
Definition Database.h:238
EDatabaseState m_configState
Current configuration state of the database.
Definition Database.h:246
EDatabaseState
State of the database.
Definition Database.h:45
@ c_PreInit
Before any initialization.
Definition Database.h:47
@ c_InitGlobaltagList
Globaltag list has been finalized metadata providers not and globaltags are not checked for usability...
Definition Database.h:49
@ c_Ready
Everything is ready.
Definition Database.h:51
Conditions::PayloadMetadata DBQuery
Struct for bulk read queries.
Definition Database.h:72
std::unique_ptr< Conditions::PayloadProvider > m_payloadProvider
The active payload provider.
Definition Database.h:240
std::set< std::string > m_usableTagStates
Set of usable globaltag states to be handed to the metadata providers.
Definition Database.h:236
std::unique_ptr< Conditions::TestingPayloadStorage > m_payloadCreation
testing payload storage to create new payloads
Definition Database.h:242
std::vector< std::string > m_globalTags
List of globaltags to be used.
Definition Database.h:234
std::vector< Conditions::TestingPayloadStorage > m_testingPayloads
optional list of testing payload storages to look for existing payloads
Definition Database.h:244
std::vector< std::string > m_metadataConfigurations
List of available metadata providers (which haven't been tried yet)
Definition Database.h:230
Database()=default
Hidden constructor, as it is a singleton.
Store event, run, and experiment numbers.
A class that describes the interval of experiments/runs for which an object in the database is valid.
Simple ScopeGuard to execute a function at the end of the object lifetime.
Definition ScopeGuard.h:36
Class to store variables with their name which were sent to the logging service.
void reset(bool keepEntries=false)
Invalidate all payloads.
Definition DBStore.cc:175
std::string getGlobalTags()
Return the global tags used by the database.
Definition Database.cc:153
void initialize(const EDatabaseState target=c_Ready)
Initialize the database connection settings on first use.
Definition Database.cc:197
~Database()
Hidden destructor, as it is a singleton.
std::pair< TObject *, IntervalOfValidity > getData(const EventMetaData &event, const std::string &name)
Request an object from the database.
Definition Database.cc:71
static Database & Instance()
Instance of a singleton Database.
Definition Database.cc:41
static DBStore & Instance()
Instance of a singleton DBStore.
Definition DBStore.cc:26
static void exposePythonAPI()
Exposes setGlobalTag function of the Database class to Python.
Definition Database.cc:272
bool storeData(const std::string &name, TObject *object, const IntervalOfValidity &iov)
Store an object in the database.
Definition Database.cc:140
void nextMetadataProvider()
Enable the next metadataprovider in the list.
Definition Database.cc:158
static void reset(bool keepConfig=false)
Reset the database instance.
Definition Database.cc:49
ScopeGuard createScopedUpdateSession()
Make sure we have efficient http pipelinging during initialize/beginRun but don't keep session alive ...
Definition Database.cc:61
void updatePayload(unsigned int revision, const IntervalOfValidity &iov, const std::string &filename, const std::string &checksum, const std::string &globaltag, const EventMetaData &event)
Update the payload information for this entry and if appropriate open the new file and extract the ob...
Abstract base class for different kinds of events.