11 #include <framework/database/PayloadProvider.h>
12 #include <framework/logging/Logger.h>
13 #include <framework/utilities/FileSystem.h>
15 #include <boost/filesystem.hpp>
16 #include <boost/algorithm/string.hpp>
20 namespace fs = boost::filesystem;
22 namespace Belle2::Conditions {
26 if (!cacheDir.empty()) {
27 m_cacheDir = {fs::absolute(cacheDir).string(),
false};
29 m_cacheDir = {fs::absolute(fs::temp_directory_path() /
"basf2-conditions").string(),
false};
31 m_locations.reserve(locations.size() + 2);
33 m_locations.emplace_back(m_cacheDir);
34 B2DEBUG(33,
"Added payload cache location" <<
LogVar(
"path", m_cacheDir.base));
36 for (
auto path : locations) {
37 boost::algorithm::trim(path);
39 B2FATAL(
"Found empty payload location in configuration. "
40 "Please make sure that the conditions database settings are correct");
43 if (
auto pos = path.find(
"://"); pos != std::string::npos) {
45 auto protocol = path.substr(0, pos);
46 boost::algorithm::to_lower(protocol);
47 if (protocol ==
"file") {
48 path = path.substr(pos + 3);
49 }
else if (protocol ==
"http" or protocol ==
"https") {
52 B2ERROR(
"Unknown protocol, only supported protocols for payload download are file, http, https" <<
LogVar(
"protocol", protocol));
57 if (!remote) path = fs::absolute(path).string();
59 B2DEBUG(33,
"Added payload search location" <<
LogVar(remote ?
"url" :
"path", path));
60 m_locations.emplace_back(PayloadLocation{path, remote});
63 m_locations.emplace_back(PayloadLocation{
"",
true});
66 bool PayloadProvider::find(PayloadMetadata& metadata)
69 return std::any_of(m_locations.begin(), m_locations.end(), [
this, &metadata](
const auto & loc) {
70 return loc.isRemote ? getRemoteFile(loc, metadata) : getLocalFile(loc, metadata);
77 for (
EDirectoryLayout structure : {EDirectoryLayout::c_hashed, EDirectoryLayout::c_flat}) {
78 auto fullPath = fs::path(loc.base) / getFilename(structure, metadata);
80 if (!fs::exists(fullPath))
continue;
82 B2DEBUG(36,
"Checking checksum for payload file" <<
LogVar(
"name", metadata.name) <<
LogVar(
"local dir", loc.base)
83 <<
LogVar(
"revision", metadata.revision) <<
LogVar(
"filename", fullPath) <<
LogVar(
"checksum", metadata.checksum));
85 if (actual == metadata.checksum) {
86 metadata.filename = fullPath.string();
87 B2DEBUG(37,
"Found matching payload file");
90 B2DEBUG(37,
"Checksum doesn't match, continue with next");
95 bool PayloadProvider::getRemoteFile(
const PayloadLocation& loc, PayloadMetadata& metadata)
98 const auto local = fs::path(m_cacheDir.base) / getFilename(EDirectoryLayout::c_hashed, metadata);
100 const bool fallback = loc.base.empty();
101 const auto base = fallback ? metadata.baseUrl : loc.base;
103 const auto url = m_downloader.joinWithSlash(base, metadata.payloadUrl);
109 auto oldUmask = umask(0);
110 ScopeGuard umaskGuard([oldUmask] {umask(oldUmask);});
111 fs::create_directories(local.parent_path());
112 }
catch (fs::filesystem_error& e) {
113 B2WARNING(
"Cannot create local payload directory" <<
LogVar(
"directory", local.parent_path())
114 <<
LogVar(
"error", e.code().message()));
119 B2DEBUG(37,
"Attempting to lock payload file for writing ..." <<
LogVar(
"filename", local));
123 if (!writelock.
lock(m_timeout,
true)) {
124 throw std::runtime_error(
"write lock failed");
128 std::fstream localStream(local.string().c_str(), std::ios::binary | std::ios::in | std::ios::out);
129 B2DEBUG(37,
"Got write lock, check for file access ...");
130 if (!localStream.good()) {
131 B2ERROR(
"Cannot open file for writing" <<
LogVar(
"filename", local) <<
LogVar(
"error", strerror(errno)));
132 throw std::runtime_error(
"cannot open file for writing????");
135 boost::filesystem::permissions(local, boost::filesystem::all_all &
136 ~(boost::filesystem::owner_exe | boost::filesystem::group_exe | boost::filesystem::others_exe));
139 B2DEBUG(37,
"Ok, check digest in case another process downloaded already...");
140 if (not m_downloader.verifyChecksum(localStream, metadata.checksum)) {
142 B2DEBUG(37,
"Still not good, download now ...");
144 if (not m_downloader.download(url, localStream, not fallback)) {
146 B2DEBUG(37,
"Payload not found ... trying next source");
150 if (not m_downloader.verifyChecksum(localStream, metadata.checksum)) {
151 B2WARNING(
"Conditions Database: checksum mismatch after download. Trying once more in a temporary file"
152 <<
LogVar(
"name", metadata.name) <<
LogVar(
"revision", metadata.revision) <<
LogVar(
"filename", local.string()));
153 throw std::runtime_error(
"checksum mismatch");
155 }
catch (std::exception& e) {
156 B2ERROR(
"Conditions Database: failure downloading url" <<
LogVar(
"url", url) <<
LogVar(
"error", e.what()));
159 B2DEBUG(37,
"Download of payload successful");
162 metadata.filename = local.string();
164 }
catch (std::exception&) {
166 return getTemporaryFile(url, metadata, not fallback);
170 std::string PayloadProvider::getFilename(EDirectoryLayout structure,
171 const PayloadMetadata& payload)
const
175 case EDirectoryLayout::c_hashed:
176 path /= payload.checksum.substr(0, 2);
177 path /= payload.name +
"_r" + std::to_string(payload.revision) +
".root";
179 case EDirectoryLayout::c_flat:
180 path /=
"dbstore_" + payload.name +
"_rev_" + std::to_string(payload.revision) +
".root";
183 return path.string();
186 bool PayloadProvider::getTemporaryFile(
const std::string& url, PayloadMetadata& metadata,
bool silentOnMissing)
189 if (
auto && it = m_temporaryFiles.find(metadata.checksum); it != m_temporaryFiles.end()) {
190 metadata.filename = it->second->getName();
193 const auto openmode = std::ios_base::binary | std::ios_base::in | std::ios_base::out | std::ios_base::trunc;
195 B2DEBUG(37,
"Trying to download into temporary file " << tmpfile->getName());
197 if (not m_downloader.download(url, *tmpfile, silentOnMissing))
return false;
198 if (not m_downloader.verifyChecksum(*tmpfile, metadata.checksum))
return false;
199 }
catch (std::exception& e) {
200 B2ERROR(
"Conditions Database: failure downloading url" <<
LogVar(
"url", url) <<
LogVar(
"error", e.what()));
203 metadata.filename = tmpfile->getName();
204 m_temporaryFiles[metadata.checksum] = std::move(tmpfile);