Belle II Software light-2406-ragdoll
FileSystem.cc
1/**************************************************************************
2 * basf2 (Belle II Analysis Software Framework) *
3 * Author: The Belle II Collaboration *
4 * *
5 * See git log for contributors and copyright holders. *
6 * This file is licensed under LGPL-3.0, see LICENSE.md. *
7 **************************************************************************/
8
9#include <framework/utilities/FileSystem.h>
10
11#include <framework/logging/Logger.h>
12
13#include <boost/algorithm/string.hpp>
14
15#include <chrono>
16#include <random>
17#include <cstring>
18#include <filesystem>
19
20//dlopen etc.
21#include <dlfcn.h>
22#include <fcntl.h>
23#include <unistd.h>
24
25#include <TMD5.h>
26#include <zlib.h>
27
28using namespace std;
29using namespace Belle2;
30namespace fs = std::filesystem;
31
32bool FileSystem::fileExists(const string& filename)
33{
34 fs::path fullPath = fs::absolute(filename);
35 return fs::exists(fullPath);
36}
37
38bool FileSystem::fileDirExists(const string& filename)
39{
40 fs::path fullPath = fs::absolute(filename);
41 fullPath.remove_filename();
42 return fs::exists(fullPath);
43}
44
45bool FileSystem::isFile(const string& filename)
46{
47 fs::path fullPath = fs::absolute(filename);
48 return (fs::exists(fullPath)) && (fs::is_regular_file(fullPath));
49}
50
51bool FileSystem::isDir(const string& filename)
52{
53 fs::path fullPath = fs::absolute(filename);
54 return (fs::exists(fullPath)) && (fs::is_directory(fullPath));
55}
56
57bool FileSystem::isSymLink(const string& filename)
58{
59 fs::path fullPath = fs::absolute(filename);
60 return (fs::exists(fullPath)) && (fs::is_symlink(fullPath));
61}
62
63bool FileSystem::loadLibrary(std::string library, bool fullname)
64{
65 if (!fullname) library = "lib" + library + ".so";
66
67 B2DEBUG(100, "Loading shared library " << library);
68 void* libPointer = dlopen(library.c_str(), RTLD_LAZY | RTLD_GLOBAL);
69
70 if (libPointer == nullptr) {
71 B2ERROR("Could not open shared library file (error in dlopen) : " << dlerror());
72 return false;
73 }
74
75 return true;
76}
77
78std::string FileSystem::calculateMD5(const std::string& filename)
79{
80 if (not isFile(filename)) return "";
81 fs::path fullPath = fs::absolute(filename);
82 std::unique_ptr<TMD5> md5(TMD5::FileChecksum(fullPath.c_str()));
83 return md5->AsString();
84}
85
86std::string FileSystem::calculateAdler32(const std::string& filename)
87{
88 string chksum;
89 if (not isFile(filename)) return "";
90 fs::path fullPath = fs::absolute(filename);
91 FILE* fp = fopen(fullPath.c_str(), "rb");
92 if (fp) {
93 uLong i, sum = adler32(0, 0, 0);
94 char hexdigest[9];
95 Bytef* buf = (Bytef*) malloc(1024 * 1024 * sizeof(Bytef));
96 if (!buf) {
97 fclose(fp);
98 return "";
99 }
100 while ((i = fread((void*) buf, 1, sizeof(buf), fp)) > 0) {
101 sum = adler32(sum, buf, i);
102 }
103 fclose(fp);
104 free(buf);
105 // Adler32 checksums hex digests ARE zero padded although
106 // HLT legacy presentation may differ.
107 sprintf(hexdigest, "%08lx", sum);
108 chksum = hexdigest;
109 } else {
110 chksum = "";
111 }
112 return chksum;
113}
114
115std::string FileSystem::findFile(const string& path, const std::vector<std::string>& dirs, bool silent)
116{
117 // check given directories
118 string fullpath;
119 for (auto dir : dirs) {
120 if (dir.empty())
121 continue;
122 fs::path dir_path = dir;
123 if (fs::path(path).is_absolute())
124 dir_path += path;
125 else
126 dir_path /= path;
127 fullpath = dir_path.string();
128 if (fileExists(fullpath)) {
129 if (isSymLink(fullpath) or isSymLink(dir))
130 return fullpath;
131 else
132 return fs::canonical(fullpath).string();
133 }
134 }
135
136 // check local directory
137 fullpath = fs::absolute(path).string();
138 if (fileExists(fullpath)) {
139 if (isSymLink(fullpath))
140 return fullpath;
141 else
142 return fs::canonical(fullpath).string();
143 }
144
145 // nothing found
146 if (!silent)
147 B2ERROR("findFile(): Could not find file." << LogVar("path", path));
148 return string("");
149}
150
151std::string FileSystem::findFile(const string& path, bool silent)
152{
153 std::vector<std::string> dirs;
154 if (getenv("BELLE2_LOCAL_DIR")) {
155 dirs.emplace_back(getenv("BELLE2_LOCAL_DIR"));
156 }
157 if (getenv("BELLE2_RELEASE_DIR")) {
158 dirs.emplace_back(getenv("BELLE2_RELEASE_DIR"));
159 }
160 return findFile(path, dirs, silent);
161}
162
163std::string FileSystem::findFile(const string& path, const std::string& dataType, bool silent)
164{
165 std::vector<std::string> dirs;
166 std::string envVar = "BELLE2_" + boost::to_upper_copy(dataType) + "_DATA_DIR";
167 if (getenv(envVar.c_str())) {
168 dirs.emplace_back(getenv(envVar.c_str()));
169 }
170 std::string result = findFile(path, dirs, true);
171 if (result.empty() && !silent)
172 B2ERROR("findFile(): Could not find data file. You may want to use the 'b2install-data' tool to get the file."
173 << LogVar("path", path) << LogVar("data type", dataType));
174 return result;
175}
176
177FileSystem::Lock::Lock(const std::string& fileName, bool readonly) :
178 m_readOnly(readonly)
179{
180 const int mode = readonly ? O_RDONLY : O_RDWR;
181 m_file = open(fileName.c_str(), mode | O_CREAT, 0640);
182}
183
185{
186 if (m_file >= 0) close(m_file);
187}
188
189bool FileSystem::Lock::lock(int timeout, bool ignoreErrors)
190{
191 if (m_file < 0) return false;
192
193 auto const maxtime = std::chrono::steady_clock::now() + std::chrono::seconds(timeout);
194 std::default_random_engine random;
195 std::uniform_int_distribution<int> uniform(1, 100);
196
197 /* Note:
198 * Previously, this used flock(), which doesn't work with GPFS.
199 * fcntl() does, and also should be more likely to work on NFS.
200 * If you use the 'nolock' mount option to NFS, you are on your own.
201 */
202 struct flock fl;
203 memset(&fl, '\0', sizeof(fl));
204 fl.l_type = m_readOnly ? F_RDLCK : F_WRLCK;
205 //lock entire file
206 fl.l_whence = SEEK_SET;
207 fl.l_start = 0;
208 fl.l_len = 0;
209
210 while (true) {
211 int lock = fcntl(m_file, F_SETLK, &fl);
212 if (lock == 0)
213 return true;
214 else if (std::chrono::steady_clock::now() > maxtime)
215 break;
216 if (errno != EAGAIN && errno != EACCES && errno != EINTR) break;
217 usleep(uniform(random) * 1000);
218 }
219 if (!ignoreErrors) B2ERROR("Locking failed: " << strerror(errno));
220 return false;
221}
222
223FileSystem::TemporaryFile::TemporaryFile(std::ios_base::openmode mode): std::fstream()
224{
225 char* temporaryFileName = strdup((std::filesystem::temp_directory_path() / "basf2_XXXXXX").c_str());
226 int fileDescriptor = mkstemp(temporaryFileName);
227 if (fileDescriptor == -1) {
228 B2ERROR("Cannot create temporary file: " << strerror(errno));
229 free(temporaryFileName);
230 return;
231 }
232 m_filename = std::string(temporaryFileName);
233 open(temporaryFileName, mode);
234 if (!is_open()) {
235 B2ERROR("Cannot open temporary file: " << strerror(errno));
236 }
237 free(temporaryFileName);
238 ::close(fileDescriptor);
239}
240
242{
243 close();
244 fs::remove(m_filename);
245}
int m_file
File descriptor of file to be locked.
Definition: FileSystem.h:123
Lock(const std::string &fileName, bool readonly=false)
Construct a Lock object for the given file.
Definition: FileSystem.cc:177
bool lock(int timeout=300, bool ignoreErrors=false)
Try to lock the file.
Definition: FileSystem.cc:189
~TemporaryFile()
close file and delete on destruction
Definition: FileSystem.cc:241
std::string m_filename
filename of the temporary file
Definition: FileSystem.h:144
TemporaryFile(std::ios_base::openmode mode=std::ios_base::trunc|std::ios_base::out)
construct a new temporary file
Definition: FileSystem.cc:223
static bool loadLibrary(std::string library, bool fullname=true)
Load a shared library.
Definition: FileSystem.cc:63
static bool isSymLink(const std::string &filename)
Check if filename points to an existing symbolic link.
Definition: FileSystem.cc:57
static bool fileDirExists(const std::string &filename)
Check if the dir containing the filename exists.
Definition: FileSystem.cc:38
static bool isFile(const std::string &filename)
Check if filename points to an existing file.
Definition: FileSystem.cc:45
static std::string findFile(const std::string &path, bool silent=false)
Search for given file or directory in local or central release directory, and return absolute path if...
Definition: FileSystem.cc:151
static std::string calculateMD5(const std::string &filename)
Calculate the MD5 checksum of a given file.
Definition: FileSystem.cc:78
static std::string calculateAdler32(const std::string &filename)
Calculate the Adler-32 checksum of a given file.
Definition: FileSystem.cc:86
static bool isDir(const std::string &filename)
Check if filename points to an existing directory.
Definition: FileSystem.cc:51
static bool fileExists(const std::string &filename)
Check if the file with given filename exists.
Definition: FileSystem.cc:32
Class to store variables with their name which were sent to the logging service.
Abstract base class for different kinds of events.
Definition: ClusterUtils.h:24
STL namespace.