Belle II Software light-2406-ragdoll
FileCatalog.cc
1/**************************************************************************
2 * basf2 (Belle II Analysis Software Framework) *
3 * Author: The Belle II Collaboration *
4 * *
5 * See git log for contributors and copyright holders. *
6 * This file is licensed under LGPL-3.0, see LICENSE.md. *
7 **************************************************************************/
8
9#include <framework/core/FileCatalog.h>
10#include <framework/dataobjects/FileMetaData.h>
11#include <framework/logging/Logger.h>
12#include <framework/utilities/FileSystem.h>
13#include <framework/utilities/EnvironmentVariables.h>
14
15#include <fstream>
16#include <filesystem>
17
18using namespace Belle2;
19namespace fs = std::filesystem;
20
21
22
24{
25 static FileCatalog instance;
26 return instance;
27}
28
29
30FileCatalog::FileCatalog() : m_fileName("")
31{
32 // check for BELLE2_FILECATALOG environment variable
33 std::string fileCatalog = EnvironmentVariables::get("BELLE2_FILECATALOG", "");
34 if (fileCatalog == "NONE") return;
35
36 // check for file catalog in home directory
37 if (fileCatalog.empty()) {
38 const std::string path{"~/Belle2FileCatalog.xml"};
39 if (fs::exists(path)) {
40 fileCatalog = path;
41 }
42 }
43
44 // use file catalog in current directory if nothing else found
45 if (fileCatalog.empty()) {
46 fileCatalog = "Belle2FileCatalog.xml";
47 }
48
49 // get absolute path name
50 m_fileName = fs::absolute(fileCatalog).c_str();
51}
52
53
55{
56 fileMap.clear();
57
58 std::ifstream file(m_fileName.c_str());
59 if (!file.is_open()) return false;
60
61 try {
62 while (!file.eof()) {
63 FileMetaData entry;
64 std::string physicalFileName;
65 if (entry.read(file, physicalFileName)) fileMap[entry.getLfn()] = std::make_pair(physicalFileName, entry);
66 }
67 } catch (std::exception& e) {
68 B2ERROR("Errors occured while reading " << m_fileName <<
69 ", maybe it is corrupted? Note that your .root files should be unaffected. (Error details: " << e.what() << ")");
70 return false;
71 }
72
73 return true;
74}
75
76
78{
79 std::ofstream file(m_fileName.c_str());
80 if (!file.is_open()) return false;
81
82 file << "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n";
83 file << "<FileCatalog>\n";
84 for (const auto& entry : fileMap) entry.second.second.write(file, entry.second.first);
85 file << "</FileCatalog>\n";
86
87 return true;
88}
89
90bool FileCatalog::registerFile(const std::string& fileName, FileMetaData& metaData, const std::string& oldLFN)
91{
92 if (m_fileName.empty()) return false;
93
94 // make sure we have an LFN
95 if (metaData.getLfn().empty()) {
96 B2ERROR("Cannot register a file without a valid LFN");
97 return false;
98 }
99
100 // get lock for write access to file catalog
102 if (!lock.lock()) {
103 B2ERROR("Locking of file catalog " << m_fileName << " failed.");
104 return false;
105 }
106
107 // read the file catalog
108 FileMap fileMap;
109 if (!readCatalog(fileMap)) {
110 B2ERROR("Failed to read file catalog " << m_fileName);
111 return false;
112 }
113
114 // check whether a file with this name is already registered and remove if so
115 for (auto it = fileMap.begin(); it != fileMap.end(); ++it) {
116 auto&& lfn = (*it).first;
117 if (!oldLFN.empty() and oldLFN == lfn) {
118 // old LFN exists and we requested an update so no warning, just remove
119 fileMap.erase(it);
120 break;
121 }
122 if (metaData.getLfn() == lfn) {
123 B2WARNING("A file with the same LFN is already registered and will be overwritten in the catalog."
124 << LogVar("LFN", lfn) << LogVar("old PFN", (*it).second.first) << LogVar("new PFN", fileName));
125 fileMap.erase(it);
126 break;
127 }
128 }
129
130 // add the new entry and write the file catalog
131 fileMap[metaData.getLfn()] = std::make_pair(fileName, metaData);
132
133 if (!writeCatalog(fileMap)) {
134 B2ERROR("Failed to write file catalog " << m_fileName);
135 return false;
136 }
137
138 return true;
139}
140
141
142bool FileCatalog::getMetaData(std::string& fileName, FileMetaData& metaData)
143{
144 metaData = FileMetaData();
145 if (m_fileName.empty()) return false;
146 if (!fs::exists(m_fileName)) return false;
147
148 // get lock for read access to file catalog
149 FileSystem::Lock lock(m_fileName, true);
150 if (!lock.lock()) {
151 B2ERROR("Locking of file catalog " << m_fileName << " failed.");
152 return false;
153 }
154
155 // read the file catalog
156 FileMap fileMap;
157 if (!readCatalog(fileMap)) {
158 B2ERROR("Failed to read file catalog " << m_fileName);
159 return false;
160 }
161
162 // find the entry with given LFN
163 auto iEntry = fileMap.find(fileName);
164 if (iEntry != fileMap.end()) {
165 metaData = iEntry->second.second;
166 if (!iEntry->second.first.empty()) fileName = iEntry->second.first;
167 return true;
168 }
169 for (const auto& entry : fileMap) {
170 if (fileName.compare(entry.second.first) == 0) {
171 metaData = entry.second.second;
172 return true;
173 }
174 }
175
176 return false;
177}
178
179
180std::string FileCatalog::getPhysicalFileName(const std::string& lfn)
181{
182 std::string fileName = lfn;
183 FileMetaData metaData;
184 if (!getMetaData(fileName, metaData)) {
185 B2DEBUG(100, "No LFN " << lfn << " found in the file catalog.");
186 }
187 return fileName;
188}
This class provides an interface to the file (metadata) catalog.
Definition: FileCatalog.h:24
static FileCatalog & Instance()
Static method to get a reference to the FileCatalog instance.
Definition: FileCatalog.cc:23
virtual bool registerFile(const std::string &fileName, FileMetaData &metaData, const std::string &oldLFN="")
Register a file in the (local) file catalog.
Definition: FileCatalog.cc:90
std::map< std::string, std::pair< std::string, FileMetaData > > FileMap
Map with file catalog content.
Definition: FileCatalog.h:71
virtual bool getMetaData(std::string &fileName, FileMetaData &metaData)
Get the metadata of a file with given (logical) file name.
Definition: FileCatalog.cc:142
virtual std::string getPhysicalFileName(const std::string &lfn)
Get the physical file name for the LFN.
Definition: FileCatalog.cc:180
std::string m_fileName
Name of the file catalog file.
Definition: FileCatalog.h:89
bool writeCatalog(const FileMap &fileMap)
Write the file catalog to the local file.
Definition: FileCatalog.cc:77
FileCatalog()
Constructor: locate local database file.
Definition: FileCatalog.cc:30
bool readCatalog(FileMap &fileMap)
Read the file catalog from the local file.
Definition: FileCatalog.cc:54
Metadata information about a file.
Definition: FileMetaData.h:29
const std::string & getLfn() const
Logical file name getter.
Definition: FileMetaData.h:37
bool read(std::istream &input, std::string &physicalFileName)
Read file meta data in xml format from the input stream.
Helper class for locking a file.
Definition: FileSystem.h:97
bool lock(int timeout=300, bool ignoreErrors=false)
Try to lock the file.
Definition: FileSystem.cc:189
Class to store variables with their name which were sent to the logging service.
static std::string get(const std::string &name, const std::string &fallback="")
Get the value of an environment variable or the given fallback value if the variable is not set.
Abstract base class for different kinds of events.
Definition: ClusterUtils.h:24