Belle II Software light-2406-ragdoll
TestingPayloadStorage.cc
1/**************************************************************************
2 * basf2 (Belle II Analysis Software Framework) *
3 * Author: The Belle II Collaboration *
4 * *
5 * See git log for contributors and copyright holders. *
6 * This file is licensed under LGPL-3.0, see LICENSE.md. *
7 **************************************************************************/
8
9#include <framework/database/TestingPayloadStorage.h>
10#include <framework/dataobjects/EventMetaData.h>
11#include <framework/logging/Logger.h>
12#include <framework/utilities/FileSystem.h>
13#include <framework/utilities/ScopeGuard.h>
14
15#include <TDirectory.h>
16#include <TFile.h>
17#include <TObject.h>
18
19#include <filesystem>
20
21#include <boost/algorithm/string.hpp>
22
23namespace fs = std::filesystem;
24
25namespace Belle2::Conditions {
27 m_filename{filename}, m_absoluteFilename{fs::absolute(m_filename).string()},
28 m_payloadDir{fs::path(m_absoluteFilename).parent_path().string()}
29 {}
30
32 {
33 // Read database file on first access
34 if (!m_initialized) read();
35 m_initialized = true;
36 // And then let's look for a payload with the name
37 auto it = m_payloads.find(info.name);
38 if (it == m_payloads.end()) return false;
39 bool found{false};
40 // and check all of them for what is the highest revision containing the event
41 for (const auto& [revision, iov] : it->second) {
42 if (iov.contains(event)) {
43 info.revision = 0;
44 info.globaltag = "temp://" + m_absoluteFilename;
45 info.baseUrl = "";
46 info.payloadUrl = "";
47 info.filename = payloadFilename(m_payloadDir, info.name, revision);
48 info.iov = iov;
49 if (!FileSystem::fileExists(info.filename)) {
50 B2FATAL("Could not find payload file specified in testing payload storage" << LogVar("storage filen", m_filename)
51 << LogVar("name", info.name) << LogVar("local revision", info.revision)
52 << LogVar("payload filename", info.filename));
53 }
54 info.checksum = FileSystem::calculateMD5(info.filename);
55 found = true;
56 break;
57 }
58 }
59 if (found) {
60 B2WARNING("Conditions: Temporary testing payload used for processing" << LogVar("storage file", m_filename)
61 << LogVar("name", info.name) << LogVar("revision", info.revision) << LogVar("iov", info.iov));
62 }
63 // Did we find something? did we?
64 return found;
65 }
66
68 {
69 m_payloads.clear();
70 if (!fs::is_regular_file(m_absoluteFilename)) {
71 B2WARNING("Given testing payload storage file doesn't exist or is not a regular file" << LogVar("storage file", m_filename));
72 return;
73 }
74 // read and parse the database content
75 std::ifstream file(m_absoluteFilename.c_str());
76 if (!file.is_open()) {
77 B2FATAL("Opening of testing payload storage file failed" << LogVar("storage file", m_filename) << LogVar("error", strerror(errno)));
78 }
79 int lineno{0};
80 try {
81 while (!file.eof()) {
82 // read the file line by line
83 std::string line;
84 std::getline(file, line);
85 ++lineno;
86 // and remove comments from the line
87 size_t commentChar = line.find('#');
88 if (commentChar != std::string::npos) {
89 line = line.substr(0, commentChar);
90 }
91 // trim whitespace on each side
92 boost::algorithm::trim(line);
93 // if nothing is left skip the line
94 if (line.empty()) continue;
95 // otherwise read name, revision and iov from the line
96 std::string name;
97 std::string revision;
99 try {
100 std::stringstream(line) >> name >> revision >> iov;
101 } catch (std::runtime_error& e) {
102 throw std::runtime_error("line must be of the form 'dbstore/<payloadname> <revision> <firstExp>,<firstRun>,<finalExp>,<finalRun>'");
103 }
104 // parse name
105 size_t pos = name.find('/');
106 if (pos == std::string::npos) {
107 throw std::runtime_error("payload name must be of the form dbstore/<payloadname>");
108 }
109 std::string module = name.substr(pos + 1, name.length());
110 // and add to map of payloads
111 B2DEBUG(39, "Found testing payload" << LogVar("storage file", m_filename) << LogVar("name", module)
112 << LogVar("revision/md5", revision) << LogVar("iov", iov));
113 m_payloads[module].emplace_back(revision, iov);
114 }
115 } catch (std::exception& e) {
116 B2FATAL("Problem reading testing payloads storage" << LogVar("storage file", m_filename)
117 << LogVar("line", lineno) << LogVar("error", e.what()));
118 }
119 // and reverse all the payloads so the last ones in the file have highest priority
120 for (auto& payload : m_payloads) {
121 std::reverse(payload.second.begin(), payload.second.end());
122 }
123 }
124
125 std::string TestingPayloadStorage::payloadFilename(const std::string& path, const std::string& name,
126 const std::string& revision)
127 {
128 std::stringstream result;
129 if (!path.empty()) result << path << '/';
130 result << "dbstore_" << name << "_rev_" << revision << ".root";
131 return result.str();
132 }
133
134 bool TestingPayloadStorage::storeData(const std::string& name, TObject* object, const IntervalOfValidity& iov)
135 {
136 return store(name, iov, "", [this, &object, &name](const std::string & filename) {
137 return writePayload(filename, name, object);
138 });
139 }
140
141 bool TestingPayloadStorage::storePayload(const std::string& name, const std::string& fileName, const IntervalOfValidity& iov)
142 {
143 // resolve all symbolic links to make sure we point to the real file
144 fs::path resolved = fs::canonical(fileName);
145 if (not fs::is_regular_file(resolved)) {
146 B2ERROR("Problem creating testing payload: Given payload storage file doesn't exist" << LogVar("storage file", fileName));
147 return false;
148 }
149 return store(name, iov, resolved.string(), [&resolved](const std::string & destination) {
150 // copy payload file to payload directory and rename it to follow the file name convention
151 fs::copy_file(resolved, destination, fs::copy_options::overwrite_existing);
152 return true;
153 });
154 }
155
156 bool TestingPayloadStorage::store(const std::string& name, const IntervalOfValidity& iov,
157 const std::string& source, const std::function<bool(const std::string&)>& writer)
158 {
159 if (iov.empty()) {
160 B2ERROR("IoV is empty, refusing to store object in testing payload storage"
161 "Please provide a valid experiment/run range for the data, for example "
162 "using IntervalOfValidity::always() to store data which is always valid"
163 << LogVar("name", name));
164 return false;
165 }
166
167 if (!fs::exists(m_payloadDir)) {
168 fs::create_directories(m_payloadDir);
169 }
170
171 // create a temporary file if we don't have a source file yet
172 fs::path sourcefile{source};
173 int length = m_payloadDir.length();
174 char* temporaryFileName = new char[length + 16];
175 std::strcpy(temporaryFileName, m_payloadDir.c_str());
176 std::strcpy(temporaryFileName + length, "/payload_XXXXXX");
177 if (source.empty()) {
178 while (true) {
179 int fileDescriptor = mkstemp(temporaryFileName);
180 if ((fileDescriptor == -1) && (errno != EINTR)) {
181 B2ERROR("Cannot create payload file:" << strerror(errno));
182 delete[] temporaryFileName;
183 return false;
184 }
185 if (fileDescriptor > 0) {
186 sourcefile = temporaryFileName;
187 close(fileDescriptor);
188 break;
189 }
190 B2DEBUG(35, "first try to create tempfile failed, trying again");
191 }
192 if (!writer(sourcefile.string())) return false;
193 }
194 delete[] temporaryFileName;
195 // If we created a temporary file we want to delete it again so we'd like to
196 // use a scope guard to do so. However we need it in this scope so we need
197 // to create one in any case and release it if we didn't create a temporary
198 // file
199 ScopeGuard delete_srcfile([&sourcefile] {fs::remove(sourcefile);});
200 if (!source.empty()) delete_srcfile.release();
201
202 std::string md5 = FileSystem::calculateMD5(sourcefile.string());
203
204 // Ok, now we have the file and it's md5 sum so let's get a write lock to the database file
205 // to avoid race conditions when creating files and writing the info in the text file.
207 if (!lock.lock()) {
208 B2ERROR("Locking of testing payload storage file failed, cannot create payload"
209 << LogVar("storage file", m_filename));
210 return false;
211 }
212 std::ofstream file(m_absoluteFilename.c_str(), std::ios::app);
213 if (!file.is_open()) {
214 B2ERROR("Could not open testing payload storage file for writing" << LogVar("storage file", m_filename));
215 }
216
217 // So let's try renaming our temporary payload file to final destination
218 // We start with a 5 digit hash and expand if there's a collision
219 std::string revision;
220 bool found = false;
221 for (int i = 6; i <= 32; ++i) {
222 revision = md5.substr(0, i);
223 auto filename = payloadFilename(m_payloadDir, name, revision);
224 if (FileSystem::fileExists(filename)) {
225 if (md5 != FileSystem::calculateMD5(filename)) continue;
226 } else {
227 if (source.empty()) {
228 fs::rename(sourcefile, filename);
229 delete_srcfile.release();
230 } else {
231 fs::copy_file(source, filename, fs::copy_options::overwrite_existing);
232 }
233 }
234 found = true;
235 break;
236 }
237 if (!found) {
238 B2ERROR("Cannot create payload file: checksum mismatch for existing files");
239 return false;
240 }
241 // Ok, add to the text file
242 file << "dbstore/" << name << " " << revision << " " << iov << std::endl;
243 B2DEBUG(32, "Storing testing payload" << LogVar("storage file", m_filename) << LogVar("name", name)
244 << LogVar("local revision", revision) << LogVar("iov", iov));
245 // And make sure we reread the file on next request to payloads
246 m_initialized = false;
247 return true;
248 }
249
250 bool TestingPayloadStorage::writePayload(const std::string& fileName, const std::string& name, const TObject* object)
251 {
252 // Save the current gDirectory
253 TDirectory::TContext saveDir;
254 // And create a reproducible TFile: one that has the same checksum every time it's created as long as the content is the same
255 std::unique_ptr<TFile> file{TFile::Open((fileName + "?reproducible=PayloadFile").c_str(), "RECREATE")};
256 if (!file || !file->IsOpen()) {
257 B2ERROR("Could not open payload file for writing." << LogVar("filename", m_filename));
258 return false;
259 }
260 // Write the payload
261 object->Write(name.c_str(), TObject::kSingleKey);
262 // Done, let's go
263 file->Close();
264 return true;
265 }
266
267} // Belle2::Conditions namespace
bool storePayload(const std::string &name, const std::string &fileName, const IntervalOfValidity &iov)
Store an existing file as payload with given name and interval of validity.
bool m_initialized
Remember whether we read the file already.
std::string m_payloadDir
Directory containing the storage file as absolute file name.
bool writePayload(const std::string &fileName, const std::string &name, const TObject *object)
Write a payload file from the given object and name.
static std::string payloadFilename(const std::string &path, const std::string &name, const std::string &revision)
Build the filename for a new payload with a given name and revision in a directory.
std::unordered_map< std::string, std::vector< std::tuple< std::string, IntervalOfValidity > > > m_payloads
Map of known payloads to a list of known revisions and their interval of validity.
void read()
Read the given storage file, done lazily on first access to get() after construction or call to reset...
bool storeData(const std::string &name, TObject *object, const IntervalOfValidity &iov)
Store a TObject instance as a payload with given name and interval of validity.
bool store(const std::string &name, const IntervalOfValidity &iov, const std::string &source, const std::function< bool(const std::string &)> &writer)
Try to store a new payload with the given name and interval of validity.
bool get(const EventMetaData &event, PayloadMetadata &info)
Try to fill the PayloadMetaData for the given EventMetaData, return true on success,...
std::string m_filename
Storage file where to look for payloads.
std::string m_absoluteFilename
Storage file where to look for payloads converted to an absolute path to be robust against directory ...
TestingPayloadStorage(const std::string &filename)
Create a new instance to work on a given filename.
Store event, run, and experiment numbers.
Definition: EventMetaData.h:33
Helper class for locking a file.
Definition: FileSystem.h:97
bool lock(int timeout=300, bool ignoreErrors=false)
Try to lock the file.
Definition: FileSystem.cc:189
static std::string calculateMD5(const std::string &filename)
Calculate the MD5 checksum of a given file.
Definition: FileSystem.cc:78
static bool fileExists(const std::string &filename)
Check if the file with given filename exists.
Definition: FileSystem.cc:32
A class that describes the interval of experiments/runs for which an object in the database is valid.
Simple ScopeGuard to execute a function at the end of the object lifetime.
Definition: ScopeGuard.h:36
void release()
Release the guard without calling the cleanup function.
Definition: ScopeGuard.h:56
Class to store variables with their name which were sent to the logging service.
Simple struct to group all information necessary for a single payload.