Belle II Software  release-05-01-25
FileCatalog.cc
1 /**************************************************************************
2  * BASF2 (Belle Analysis Framework 2) *
3  * Copyright(C) 2010-2013 Belle II Collaboration *
4  * *
5  * Author: The Belle II Collaboration *
6  * Contributors: Thomas Kuhr *
7  * *
8  * This software is provided "as is" without any warranty. *
9  **************************************************************************/
10 
11 #include <framework/core/FileCatalog.h>
12 #include <framework/dataobjects/FileMetaData.h>
13 #include <framework/logging/Logger.h>
14 #include <framework/utilities/FileSystem.h>
15 #include <framework/utilities/EnvironmentVariables.h>
16 #include <boost/filesystem.hpp>
17 #include <fstream>
18 
19 
20 using namespace Belle2;
21 namespace fs = boost::filesystem;
22 
23 
24 
26 {
27  static FileCatalog instance;
28  return instance;
29 }
30 
31 
32 FileCatalog::FileCatalog() : m_fileName("")
33 {
34  // check for BELLE2_FILECATALOG environment variable
35  std::string fileCatalog = EnvironmentVariables::get("BELLE2_FILECATALOG", "");
36  if (fileCatalog == "NONE") return;
37 
38  // check for file catalog in home directory
39  if (fileCatalog.empty()) {
40  const std::string path{"~/Belle2FileCatalog.xml"};
41  if (fs::exists(path)) {
42  fileCatalog = path;
43  }
44  }
45 
46  // use file catalog in current directory if nothing else found
47  if (fileCatalog.empty()) {
48  fileCatalog = "Belle2FileCatalog.xml";
49  }
50 
51  // get absolute path name
52  m_fileName = fs::absolute(fileCatalog, fs::initial_path<fs::path>()).c_str();
53 }
54 
55 
57 {
58  fileMap.clear();
59 
60  std::ifstream file(m_fileName.c_str());
61  if (!file.is_open()) return false;
62 
63  try {
64  while (!file.eof()) {
65  FileMetaData entry;
66  std::string physicalFileName;
67  if (entry.read(file, physicalFileName)) fileMap[entry.getLfn()] = std::make_pair(physicalFileName, entry);
68  }
69  } catch (std::exception& e) {
70  B2ERROR("Errors occured while reading " << m_fileName <<
71  ", maybe it is corrupted? Note that your .root files should be unaffected. (Error details: " << e.what() << ")");
72  return false;
73  }
74 
75  return true;
76 }
77 
78 
80 {
81  std::ofstream file(m_fileName.c_str());
82  if (!file.is_open()) return false;
83 
84  file << "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n";
85  file << "<FileCatalog>\n";
86  for (const auto& entry : fileMap) entry.second.second.write(file, entry.second.first);
87  file << "</FileCatalog>\n";
88 
89  return true;
90 }
91 
92 bool FileCatalog::registerFile(const std::string& fileName, FileMetaData& metaData, const std::string& oldLFN)
93 {
94  if (m_fileName.empty()) return false;
95 
96  // make sure we have an LFN
97  if (metaData.getLfn().empty()) {
98  B2ERROR("Cannot register a file without a valid LFN");
99  return false;
100  }
101 
102  // get lock for write access to file catalog
104  if (!lock.lock()) {
105  B2ERROR("Locking of file catalog " << m_fileName << " failed.");
106  return false;
107  }
108 
109  // read the file catalog
110  FileMap fileMap;
111  if (!readCatalog(fileMap)) {
112  B2ERROR("Failed to read file catalog " << m_fileName);
113  return false;
114  }
115 
116  // check whether a file with this name is already registered and remove if so
117  for (auto it = fileMap.begin(); it != fileMap.end(); ++it) {
118  auto&& [lfn, value] = *it;
119  if (!oldLFN.empty() and oldLFN == lfn) {
120  // old LFN exists and we requested an update so no warning, just remove
121  fileMap.erase(it);
122  break;
123  }
124  if (metaData.getLfn() == lfn) {
125  B2WARNING("A file with the same LFN is already registered and will be overwritten in the catalog."
126  << LogVar("LFN", lfn) << LogVar("old PFN", value.first) << LogVar("new PFN", fileName));
127  fileMap.erase(it);
128  break;
129  }
130  }
131 
132  // add the new entry and write the file catalog
133  fileMap[metaData.getLfn()] = std::make_pair(fileName, metaData);
134 
135  if (!writeCatalog(fileMap)) {
136  B2ERROR("Failed to write file catalog " << m_fileName);
137  return false;
138  }
139 
140  return true;
141 }
142 
143 
144 bool FileCatalog::getMetaData(std::string& fileName, FileMetaData& metaData)
145 {
146  metaData = FileMetaData();
147  if (m_fileName.empty()) return false;
148  if (!fs::exists(m_fileName)) return false;
149 
150  // get lock for read access to file catalog
151  FileSystem::Lock lock(m_fileName, true);
152  if (!lock.lock()) {
153  B2ERROR("Locking of file catalog " << m_fileName << " failed.");
154  return false;
155  }
156 
157  // read the file catalog
158  FileMap fileMap;
159  if (!readCatalog(fileMap)) {
160  B2ERROR("Failed to read file catalog " << m_fileName);
161  return false;
162  }
163 
164  // find the entry with given LFN
165  auto iEntry = fileMap.find(fileName);
166  if (iEntry != fileMap.end()) {
167  metaData = iEntry->second.second;
168  if (!iEntry->second.first.empty()) fileName = iEntry->second.first;
169  return true;
170  }
171  for (const auto& entry : fileMap) {
172  if (fileName.compare(entry.second.first) == 0) {
173  metaData = entry.second.second;
174  return true;
175  }
176  }
177 
178  return false;
179 }
180 
181 
182 std::string FileCatalog::getPhysicalFileName(const std::string& lfn)
183 {
184  std::string fileName = lfn;
185  FileMetaData metaData;
186  if (!getMetaData(fileName, metaData)) {
187  B2DEBUG(100, "No LFN " << lfn << " found in the file catalog.");
188  }
189  return fileName;
190 }
Belle2::FileSystem::Lock::lock
bool lock(int timeout=300, bool ignoreErrors=false)
Try to lock the file.
Definition: FileSystem.cc:190
Belle2::FileMetaData::read
bool read(std::istream &input, std::string &physicalFileName)
Read file meta data in xml format from the input stream.
Definition: FileMetaData.cc:109
Belle2::FileCatalog::Instance
static FileCatalog & Instance()
Static method to get a reference to the FileCatalog instance.
Definition: FileCatalog.cc:25
Belle2::FileCatalog
This class provides an interface to the file (metadata) catalog.
Definition: FileCatalog.h:34
Belle2::FileCatalog::getPhysicalFileName
virtual std::string getPhysicalFileName(const std::string &lfn)
Get the physical file name for the LFN.
Definition: FileCatalog.cc:182
Belle2::FileCatalog::getMetaData
virtual bool getMetaData(std::string &fileName, FileMetaData &metaData)
Get the metadata of a file with given (logical) file name.
Definition: FileCatalog.cc:144
Belle2::FileCatalog::m_fileName
std::string m_fileName
Name of the file catalog file.
Definition: FileCatalog.h:99
Belle2::EnvironmentVariables::get
static std::string get(const std::string &name, const std::string &fallback="")
Get the value of an environment variable or the given fallback value if the variable is not set.
Definition: EnvironmentVariables.cc:35
Belle2::FileMetaData
Metadata information about a file.
Definition: FileMetaData.h:39
Belle2
Abstract base class for different kinds of events.
Definition: MillepedeAlgorithm.h:19
LogVar
Class to store variables with their name which were sent to the logging service.
Definition: LogVariableStream.h:24
Belle2::FileCatalog::writeCatalog
bool writeCatalog(const FileMap &fileMap)
Write the file catalog to the local file.
Definition: FileCatalog.cc:79
Belle2::FileSystem::Lock
Helper class for locking a file.
Definition: FileSystem.h:107
Belle2::FileCatalog::registerFile
virtual bool registerFile(const std::string &fileName, FileMetaData &metaData, const std::string &oldLFN="")
Register a file in the (local) file catalog.
Definition: FileCatalog.cc:92
Belle2::FileCatalog::readCatalog
bool readCatalog(FileMap &fileMap)
Read the file catalog from the local file.
Definition: FileCatalog.cc:56
Belle2::FileCatalog::FileCatalog
FileCatalog()
Constructor: locate local database file.
Definition: FileCatalog.cc:32
Belle2::FileMetaData::getLfn
const std::string & getLfn() const
Logical file name getter.
Definition: FileMetaData.h:47
Belle2::FileCatalog::FileMap
std::map< std::string, std::pair< std::string, FileMetaData > > FileMap
Map with file catalog content.
Definition: FileCatalog.h:81