Belle II Software light-2406-ragdoll
Database.cc
1/**************************************************************************
2 * basf2 (Belle II Analysis Software Framework) *
3 * Author: The Belle II Collaboration *
4 * *
5 * See git log for contributors and copyright holders. *
6 * This file is licensed under LGPL-3.0, see LICENSE.md. *
7 **************************************************************************/
8
9#include <boost/python/def.hpp>
10#include <boost/python/overloads.hpp>
11#include <boost/python/docstring_options.hpp>
12#include <boost/python/list.hpp>
13#include <boost/python/dict.hpp>
14#include <boost/python/extract.hpp>
15#include <boost/python/return_value_policy.hpp>
16#include <boost/python/manage_new_object.hpp>
17#include <boost/python/raw_function.hpp>
18#include <boost/algorithm/string.hpp>
19
20#include <framework/database/Database.h>
21
22#include <framework/dataobjects/EventMetaData.h>
23#include <framework/logging/Logger.h>
24#include <framework/database/DBStore.h>
25#include <framework/database/PayloadFile.h>
26
27#include <framework/database/PayloadProvider.h>
28#include <framework/database/MetadataProvider.h>
29#include <framework/database/LocalMetadataProvider.h>
30#include <framework/database/CentralMetadataProvider.h>
31#include <framework/database/Configuration.h>
32
33#include <algorithm>
34#include <cstdlib>
35
36namespace Belle2 {
43 {
44 static Database instance;
45 return instance;
46 }
47
48 Database::~Database() = default;
49
50 void Database::reset(bool keepConfig)
51 {
53 conf.setInitialized(false);
58 if (not keepConfig)
59 conf.reset();
60 }
61
63 {
64 // make sure we reread testing text files in case they got updated
65 for (auto& testing : m_testingPayloads) {
66 testing.reset();
67 }
68 // and return a downloader session guard for the downloader we use
70 }
71
72 std::pair<TObject*, IntervalOfValidity> Database::getData(const EventMetaData& event, const std::string& name)
73 {
74 DBStoreEntry entry(DBStoreEntry::c_Object, name, TObject::Class(), false, true);
75 std::vector<DBQuery> query{DBQuery{name, true}};
76 getData(event, query);
77 entry.updatePayload(query[0].revision, query[0].iov, query[0].filename, query[0].checksum, query[0].globaltag, event);
78 return std::make_pair(entry.releaseObject(), query[0].iov);
79 }
80
81 bool Database::getData(const EventMetaData& event, std::vector<DBQuery>& query)
82 {
83 // initialize lazily ...
85 // So first go over the requested payloads once, reset the info and check for any
86 // testing payloads we might want to use
87 const size_t testingPayloads = std::count_if(query.begin(), query.end(), [this, &event](auto & payload) {
88 // make sure the queries are "reset" to invalid revision and no filename before we start looking
89 payload.filename = "";
90 payload.revision = 0;
91 // and now look in all testing payload providers if any.
92 for (auto& tmp : m_testingPayloads) {
93 if (tmp.get(event, payload)) return true;
94 }
95 return false;
96 });
97 // if we already found all just return here
98 if (testingPayloads == query.size()) return true;
99 // nooow, lets look for proper payloads;
100 try {
101 m_metadataProvider->getPayloads(event.getExperiment(), event.getRun(), query);
102 } catch (std::exception&) {
103 // something went wrong with the metadata update ... so let's try next provider
104 B2WARNING("Conditions data: Problem with payload metadata provider, trying to fall back to next provider..."
105 << LogVar("provider", m_currentProvider));
106 nextMetadataProvider();
107 return getData(event, query);
108 }
109 // and if we could find the metadata lets also locate the payloads ...
110 const size_t payloadsLocated = std::count_if(query.begin(), query.end(), [this](auto & payload) {
111 // make sure we don't overwrite local payloads or otherwise already valid filenames;
112 if (!payload.filename.empty()) return true;
113 // but don't check for payloads we could not find. But this is only a
114 // problem if they are required so report success for not required
115 // payloads
116 if (payload.revision == 0) return not payload.required;
117 // and locate the payload.
118 if (not m_payloadProvider->find(payload)) {
119 // if that fails lets let the user know: Even for optional payloads, if
120 // we know the metadata but cannot find the file something is fishy and
121 // should be reported.
122 auto loglevel = payload.required ? LogConfig::c_Error : LogConfig::c_Warning;
123 B2LOG(loglevel, 0, "Conditions data: Could not find file for payload"
124 << LogVar("name", payload.name) << LogVar("revision", payload.revision)
125 << LogVar("checksum", payload.checksum) << LogVar("globaltag", payload.globaltag));
126 return not payload.required;
127 }
128 return true;
129 });
130 // did we find all payloads?
131 return payloadsLocated == query.size();
132 }
133
134 bool Database::storeData(std::list<DBImportQuery>& query)
135 {
136 return std::all_of(query.begin(), query.end(), [this](const auto & import) {
137 return storeData(import.name, import.object, import.iov);
138 });
139 }
140
141 bool Database::storeData(const std::string& name, TObject* obj, const IntervalOfValidity& iov)
142 {
143 if (!m_payloadCreation) initialize();
144 auto result = m_payloadCreation->storeData(name, obj, iov);
145 // we added payloads, make sure we reread testing files on next try
146 if (result) {
147 for (auto& testing : m_testingPayloads) {
148 testing.reset();
149 }
150 }
151 return result;
152 }
153
154 std::string Database::getGlobalTags()
155 {
156 return boost::algorithm::join(m_globalTags, ",");
157 }
158
159 void Database::nextMetadataProvider()
160 {
161 if (m_metadataConfigurations.empty()) {
162 B2FATAL("Conditions data: No more metadata providers available");
163 }
164 m_currentProvider = m_metadataConfigurations.back();
165 m_metadataConfigurations.pop_back();
166 bool remote{false};
167 if (auto pos = m_currentProvider.find("://"); pos != std::string::npos) {
168 // found a protocol: if file remove, otherwise keep as is and set as remote ...
169 auto protocol = m_currentProvider.substr(0, pos);
170 boost::algorithm::to_lower(protocol);
171 if (protocol == "file") {
172 m_currentProvider = m_currentProvider.substr(pos + 3);
173 } else if (protocol == "http" or protocol == "https") {
174 remote = true;
175 } else {
176 B2FATAL("Conditions data: Unknown metadata protocol, only supported protocols for payload metadata are file, http, https"
177 << LogVar("protocol", protocol));
178 }
179 }
180 try {
181 if (remote) {
182 m_metadataProvider = std::make_unique<Conditions::CentralMetadataProvider>(m_currentProvider, m_usableTagStates);
183 } else {
184 m_metadataProvider = std::make_unique<Conditions::LocalMetadataProvider>(m_currentProvider, m_usableTagStates);
185 }
186 } catch (std::exception& e) {
187 B2WARNING("Conditions data: Metadata provider not usable, trying next one ..."
188 << LogVar("provider", m_currentProvider) << LogVar("error", e.what()));
189 return nextMetadataProvider();
190 }
191 // and check the tags are useable
192 if (!m_metadataProvider->setTags(m_globalTags)) {
193 B2FATAL("Conditions data: Problems with globaltag configuration, cannot continue");
194 }
195 }
196
197 void Database::initialize(const EDatabaseState target)
198 {
199 if (m_configState >= target) return;
200 auto conf = Conditions::Configuration::getInstance();
201
202 if (m_configState == c_PreInit) {
203 // first step: freeze the configuration object and determine the list of globaltags
204 // this calculates if tag replay is possible and will create an error otherwise but
205 // it will not do anything else than setting the final list of globaltags
206 conf.setInitialized(true);
207 m_globalTags = conf.getFinalListOfTags();
208 // trim the globaltag names for removing leading/trailing whitespaces, since they
209 // are a potential source of FATAL errors
210 std::transform(m_globalTags.begin(), m_globalTags.end(), m_globalTags.begin(),
211 [](const auto & tag) {
212 return boost::algorithm::trim_copy(tag);
213 });
214 // and remove duplicates, there's no need to look in the same gt multiple times
215 std::set<std::string> seen;
216 m_globalTags.erase(std::remove_if(m_globalTags.begin(), m_globalTags.end(),
217 [&seen](const auto & tag) {
218 return not seen.insert(tag).second;
219 }), m_globalTags.end());
220 // and also obtain usable tag states and metadata providers
221 m_usableTagStates = conf.getUsableTagStates();
222 m_metadataConfigurations = conf.getMetadataProviders();
223 // reverse because we want to pop out elements when used
224 std::reverse(m_metadataConfigurations.begin(), m_metadataConfigurations.end());
225 m_currentProvider = "";
226 m_configState = c_InitGlobaltagList;
227 }
228 // do we want to stop early?
229 if (m_configState >= target) return;
230 if (m_configState == c_InitGlobaltagList) {
231 // setup the first working provider;
232 if (m_metadataConfigurations.empty()) {
233 m_metadataProvider = std::make_unique<Conditions::NullMetadataProvider>();
234 } else {
235 nextMetadataProvider();
236 }
237 // we will actually use the globaltags so print them now
238 if (!m_globalTags.empty()) {
239 // Globaltags are useable so print out the final list we're gonna use
240 if (m_globalTags.size() == 1) {
241 B2INFO("Conditions data: configured globaltag is " << m_globalTags[0]);
242 } else {
243 B2INFO("Conditions data: configured globaltags (highest priority first) are " << boost::algorithm::join(m_globalTags, ", "));
244 }
245 }
246 // Configure payload location/download
247 m_payloadProvider = std::make_unique<Conditions::PayloadProvider>(
248 conf.getPayloadLocations(),
249 conf.getDownloadCacheDirectory(),
250 conf.getDownloadLockTimeout()
251 );
252 // Also we need to be able to create payloads ...
253 m_payloadCreation = std::make_unique<Conditions::TestingPayloadStorage>(conf.getNewPayloadLocation());
254 // And maaaybe we want to use testing payloads
255 m_testingPayloads.clear();
256 for (const auto& path : conf.getTestingPayloadLocations()) {
257 B2INFO("Conditions data: configured to use testing payloads" << LogVar("location", path));
258 m_testingPayloads.emplace_back(path);
259 }
260 // If so, warn again ... because
261 if (not m_testingPayloads.empty()) {
262 B2WARNING(R"(Conditions data: configured to look for temporary testing payloads from one or more local folders.
263
264 This will lead to non-reproducible results and is strictly only for testing purposes.
265 It is NOT ALLOWED for official productions or data analysis and any results obtained like this WILL NOT BE PUBLISHED.
266 )");
267 }
268 m_configState = c_Ready;
269 }
270 }
271
272 void Database::exposePythonAPI()
273 {
274 // To avoid confusion between std::arg and boost::python::arg we want a shorthand namespace as well
275 namespace py = boost::python;
276
277 // Make sure the default instance is created
278 Database::Instance();
279
280 // Don't show c++ signature in python doc to keep it simple
281 py::docstring_options options(true, true, false);
282
283 // Expose our shiny configuration object
284 Conditions::Configuration::exposePythonAPI();
285 }
287}
static Configuration & getInstance()
Get a reference to the instance which will be used when the Database is initialized.
ScopeGuard ensureSession()
Make sure there's an active session and return a ScopeGuard object that closes the session on destruc...
Definition: Downloader.h:43
static Downloader & getDefaultInstance()
Return the default instance.
Definition: Downloader.cc:134
Class to hold one entry from the ConditionsDB in the DBStore.
Definition: DBStoreEntry.h:47
@ c_Object
A ROOT file containing a object with the name of the DBStoreEntry.
Definition: DBStoreEntry.h:56
TObject * releaseObject()
Return the pointer to the current object and release ownership: The caller is responsible to clean up...
Definition: DBStoreEntry.h:161
Singleton base class for the low-level interface to the database.
Definition: Database.h:42
std::unique_ptr< Conditions::MetadataProvider > m_metadataProvider
Currently active metadata provider.
Definition: Database.h:238
EDatabaseState m_configState
Current configuration state of the database.
Definition: Database.h:246
EDatabaseState
State of the database.
Definition: Database.h:45
@ c_PreInit
Before any initialization.
Definition: Database.h:47
std::unique_ptr< Conditions::TestingPayloadStorage > m_payloadCreation
testing payload storage to create new payloads
Definition: Database.h:242
std::vector< Conditions::TestingPayloadStorage > m_testingPayloads
optional list of testing payload storages to look for existing payloads
Definition: Database.h:244
Store event, run, and experiment numbers.
Definition: EventMetaData.h:33
A class that describes the interval of experiments/runs for which an object in the database is valid.
Simple ScopeGuard to execute a function at the end of the object lifetime.
Definition: ScopeGuard.h:36
Class to store variables with their name which were sent to the logging service.
void reset(bool keepEntries=false)
Invalidate all payloads.
Definition: DBStore.cc:177
void initialize(const EDatabaseState target=c_Ready)
Initialize the database connection settings on first use.
Definition: Database.cc:197
~Database()
Hidden destructor, as it is a singleton.
std::pair< TObject *, IntervalOfValidity > getData(const EventMetaData &event, const std::string &name)
Request an object from the database.
Definition: Database.cc:72
static Database & Instance()
Instance of a singleton Database.
Definition: Database.cc:42
static DBStore & Instance()
Instance of a singleton DBStore.
Definition: DBStore.cc:28
static void reset(bool keepConfig=false)
Reset the database instance.
Definition: Database.cc:50
ScopeGuard createScopedUpdateSession()
Make sure we have efficient http pipelinging during initialize/beginRun but don't keep session alive ...
Definition: Database.cc:62
void updatePayload(unsigned int revision, const IntervalOfValidity &iov, const std::string &filename, const std::string &checksum, const std::string &globaltag, const EventMetaData &event)
Update the payload information for this entry and if appropriate open the new file and extract the ob...
Definition: DBStoreEntry.cc:83
Abstract base class for different kinds of events.
Definition: ClusterUtils.h:24
Simple struct to group all information necessary for a single payload.