| File: | framework/utilities/src/FileSystem.cc |
| Warning: | line 100, column 17 Read function called when stream is in EOF state. Function has no effect |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
| 1 | /************************************************************************** | |||
| 2 | * basf2 (Belle II Analysis Software Framework) * | |||
| 3 | * Author: The Belle II Collaboration * | |||
| 4 | * * | |||
| 5 | * See git log for contributors and copyright holders. * | |||
| 6 | * This file is licensed under LGPL-3.0, see LICENSE.md. * | |||
| 7 | **************************************************************************/ | |||
| 8 | ||||
| 9 | #include <framework/utilities/FileSystem.h> | |||
| 10 | ||||
| 11 | #include <framework/logging/Logger.h> | |||
| 12 | ||||
| 13 | #include <boost/algorithm/string.hpp> | |||
| 14 | ||||
| 15 | #include <chrono> | |||
| 16 | #include <random> | |||
| 17 | #include <cstring> | |||
| 18 | #include <filesystem> | |||
| 19 | ||||
| 20 | //dlopen etc. | |||
| 21 | #include <dlfcn.h> | |||
| 22 | #include <fcntl.h> | |||
| 23 | #include <unistd.h> | |||
| 24 | ||||
| 25 | #include <TMD5.h> | |||
| 26 | #include <zlib.h> | |||
| 27 | ||||
| 28 | using namespace std; | |||
| 29 | using namespace Belle2; | |||
| 30 | namespace fs = std::filesystem; | |||
| 31 | ||||
| 32 | bool FileSystem::fileExists(const string& filename) | |||
| 33 | { | |||
| 34 | fs::path fullPath = fs::absolute(filename); | |||
| 35 | return fs::exists(fullPath); | |||
| 36 | } | |||
| 37 | ||||
| 38 | bool FileSystem::fileDirExists(const string& filename) | |||
| 39 | { | |||
| 40 | fs::path fullPath = fs::absolute(filename); | |||
| 41 | fullPath.remove_filename(); | |||
| 42 | return fs::exists(fullPath); | |||
| 43 | } | |||
| 44 | ||||
| 45 | bool FileSystem::isFile(const string& filename) | |||
| 46 | { | |||
| 47 | fs::path fullPath = fs::absolute(filename); | |||
| 48 | return (fs::exists(fullPath)) && (fs::is_regular_file(fullPath)); | |||
| 49 | } | |||
| 50 | ||||
| 51 | bool FileSystem::isDir(const string& filename) | |||
| 52 | { | |||
| 53 | fs::path fullPath = fs::absolute(filename); | |||
| 54 | return (fs::exists(fullPath)) && (fs::is_directory(fullPath)); | |||
| 55 | } | |||
| 56 | ||||
| 57 | bool FileSystem::isSymLink(const string& filename) | |||
| 58 | { | |||
| 59 | fs::path fullPath = fs::absolute(filename); | |||
| 60 | return (fs::exists(fullPath)) && (fs::is_symlink(fullPath)); | |||
| 61 | } | |||
| 62 | ||||
| 63 | bool FileSystem::loadLibrary(std::string library, bool fullname) | |||
| 64 | { | |||
| 65 | if (!fullname) library = "lib" + library + ".so"; | |||
| 66 | ||||
| 67 | B2DEBUG(100, "Loading shared library " << library)do { if (Belle2::LogSystem::debugEnabled()) do { if (Belle2:: LogSystem::Instance().isLevelEnabled(Belle2::LogConfig::c_Debug , 100, "framework")) { { LogVariableStream varStream; varStream << "Loading shared library " << library; Belle2:: LogSystem::Instance().sendMessage(Belle2::LogMessage(Belle2:: LogConfig::c_Debug, std::move(varStream), "framework", __PRETTY_FUNCTION__ , "framework/utilities/src/FileSystem.cc", 67, 100)); }; } } while (false); } while(false); | |||
| 68 | void* libPointer = dlopen(library.c_str(), RTLD_LAZY0x00001 | RTLD_GLOBAL0x00100); | |||
| 69 | ||||
| 70 | if (libPointer == nullptr) { | |||
| 71 | B2ERROR("Could not open shared library file (error in dlopen) : " << dlerror())do { { LogVariableStream varStream; varStream << "Could not open shared library file (error in dlopen) : " << dlerror(); Belle2::LogSystem::Instance().sendMessage (Belle2::LogMessage(Belle2::LogConfig::c_Error, std::move(varStream ), "framework", __PRETTY_FUNCTION__, "framework/utilities/src/FileSystem.cc" , 71, 0)); } } while(false); | |||
| 72 | return false; | |||
| 73 | } | |||
| 74 | ||||
| 75 | return true; | |||
| 76 | } | |||
| 77 | ||||
| 78 | std::string FileSystem::calculateMD5(const std::string& filename) | |||
| 79 | { | |||
| 80 | if (not isFile(filename)) return ""; | |||
| 81 | fs::path fullPath = fs::absolute(filename); | |||
| 82 | std::unique_ptr<TMD5> md5(TMD5::FileChecksum(fullPath.c_str())); | |||
| 83 | return md5->AsString(); | |||
| 84 | } | |||
| 85 | ||||
| 86 | std::string FileSystem::calculateAdler32(const std::string& filename) | |||
| 87 | { | |||
| 88 | string chksum; | |||
| 89 | if (not isFile(filename)) return ""; | |||
| ||||
| 90 | fs::path fullPath = fs::absolute(filename); | |||
| 91 | FILE* fp = fopen(fullPath.c_str(), "rb"); | |||
| 92 | if (fp
| |||
| 93 | uLong i, sum = adler32(0, 0, 0); | |||
| 94 | char hexdigest[9]; | |||
| 95 | Bytef* buf = (Bytef*) malloc(1024 * 1024 * sizeof(Bytef)); | |||
| 96 | if (!buf) { | |||
| 97 | fclose(fp); | |||
| 98 | return ""; | |||
| 99 | } | |||
| 100 | while ((i = fread((void*) buf, 1, sizeof(buf), fp)) > 0) { | |||
| ||||
| 101 | if (!i
| |||
| 102 | sum = adler32(sum, buf, i); | |||
| 103 | } | |||
| 104 | fclose(fp); | |||
| 105 | free(buf); | |||
| 106 | // Adler32 checksums hex digests ARE zero padded although | |||
| 107 | // HLT legacy presentation may differ. | |||
| 108 | sprintf(hexdigest, "%08lx", sum); | |||
| 109 | chksum = hexdigest; | |||
| 110 | } else { | |||
| 111 | chksum = ""; | |||
| 112 | } | |||
| 113 | return chksum; | |||
| 114 | } | |||
| 115 | ||||
| 116 | std::string FileSystem::findFile(const string& path, const std::vector<std::string>& dirs, bool silent) | |||
| 117 | { | |||
| 118 | // check given directories | |||
| 119 | string fullpath; | |||
| 120 | for (auto dir : dirs) { | |||
| 121 | if (dir.empty()) | |||
| 122 | continue; | |||
| 123 | fs::path dir_path = dir; | |||
| 124 | if (fs::path(path).is_absolute()) | |||
| 125 | dir_path += path; | |||
| 126 | else | |||
| 127 | dir_path /= path; | |||
| 128 | fullpath = dir_path.string(); | |||
| 129 | if (fileExists(fullpath)) { | |||
| 130 | if (isSymLink(fullpath) or isSymLink(dir)) | |||
| 131 | return fullpath; | |||
| 132 | else | |||
| 133 | return fs::canonical(fullpath).string(); | |||
| 134 | } | |||
| 135 | } | |||
| 136 | ||||
| 137 | // check local directory | |||
| 138 | fullpath = fs::absolute(path).string(); | |||
| 139 | if (fileExists(fullpath)) { | |||
| 140 | if (isSymLink(fullpath)) | |||
| 141 | return fullpath; | |||
| 142 | else | |||
| 143 | return fs::canonical(fullpath).string(); | |||
| 144 | } | |||
| 145 | ||||
| 146 | // nothing found | |||
| 147 | if (!silent) | |||
| 148 | B2ERROR("findFile(): Could not find file." << LogVar("path", path))do { { LogVariableStream varStream; varStream << "findFile(): Could not find file." << LogVar("path", path); Belle2::LogSystem::Instance() .sendMessage(Belle2::LogMessage(Belle2::LogConfig::c_Error, std ::move(varStream), "framework", __PRETTY_FUNCTION__, "framework/utilities/src/FileSystem.cc" , 148, 0)); } } while(false); | |||
| 149 | return string(""); | |||
| 150 | } | |||
| 151 | ||||
| 152 | std::string FileSystem::findFile(const string& path, bool silent) | |||
| 153 | { | |||
| 154 | std::vector<std::string> dirs; | |||
| 155 | if (getenv("BELLE2_LOCAL_DIR")) { | |||
| 156 | dirs.emplace_back(getenv("BELLE2_LOCAL_DIR")); | |||
| 157 | } | |||
| 158 | if (getenv("BELLE2_RELEASE_DIR")) { | |||
| 159 | dirs.emplace_back(getenv("BELLE2_RELEASE_DIR")); | |||
| 160 | } | |||
| 161 | return findFile(path, dirs, silent); | |||
| 162 | } | |||
| 163 | ||||
| 164 | std::string FileSystem::findFile(const string& path, const std::string& dataType, bool silent) | |||
| 165 | { | |||
| 166 | std::vector<std::string> dirs; | |||
| 167 | std::string envVar = "BELLE2_" + boost::to_upper_copy(dataType) + "_DATA_DIR"; | |||
| 168 | if (getenv(envVar.c_str())) { | |||
| 169 | dirs.emplace_back(getenv(envVar.c_str())); | |||
| 170 | } | |||
| 171 | std::string result = findFile(path, dirs, true); | |||
| 172 | if (result.empty() && !silent) | |||
| 173 | B2ERROR("findFile(): Could not find data file. You may want to use the 'b2install-data' tool to get the file."do { { LogVariableStream varStream; varStream << "findFile(): Could not find data file. You may want to use the 'b2install-data' tool to get the file." << LogVar("path", path) << LogVar("data type", dataType ); Belle2::LogSystem::Instance().sendMessage(Belle2::LogMessage (Belle2::LogConfig::c_Error, std::move(varStream), "framework" , __PRETTY_FUNCTION__, "framework/utilities/src/FileSystem.cc" , 174, 0)); } } while(false) | |||
| 174 | << LogVar("path", path) << LogVar("data type", dataType))do { { LogVariableStream varStream; varStream << "findFile(): Could not find data file. You may want to use the 'b2install-data' tool to get the file." << LogVar("path", path) << LogVar("data type", dataType ); Belle2::LogSystem::Instance().sendMessage(Belle2::LogMessage (Belle2::LogConfig::c_Error, std::move(varStream), "framework" , __PRETTY_FUNCTION__, "framework/utilities/src/FileSystem.cc" , 174, 0)); } } while(false); | |||
| 175 | return result; | |||
| 176 | } | |||
| 177 | ||||
| 178 | FileSystem::Lock::Lock(const std::string& fileName, bool readonly) : | |||
| 179 | m_readOnly(readonly) | |||
| 180 | { | |||
| 181 | const int mode = readonly ? O_RDONLY00 : O_RDWR02; | |||
| 182 | m_file = open(fileName.c_str(), mode | O_CREAT0100, 0640); | |||
| 183 | } | |||
| 184 | ||||
| 185 | FileSystem::Lock::~Lock() | |||
| 186 | { | |||
| 187 | if (m_file >= 0) close(m_file); | |||
| 188 | } | |||
| 189 | ||||
| 190 | bool FileSystem::Lock::lock(int timeout, bool ignoreErrors) | |||
| 191 | { | |||
| 192 | if (m_file < 0) return false; | |||
| 193 | ||||
| 194 | auto const maxtime = std::chrono::steady_clock::now() + std::chrono::seconds(timeout); | |||
| 195 | std::default_random_engine random; | |||
| 196 | std::uniform_int_distribution<int> uniform(1, 100); | |||
| 197 | ||||
| 198 | /* Note: | |||
| 199 | * Previously, this used flock(), which doesn't work with GPFS. | |||
| 200 | * fcntl() does, and also should be more likely to work on NFS. | |||
| 201 | * If you use the 'nolock' mount option to NFS, you are on your own. | |||
| 202 | */ | |||
| 203 | struct flock fl; | |||
| 204 | memset(&fl, '\0', sizeof(fl)); | |||
| 205 | fl.l_type = m_readOnly ? F_RDLCK0 : F_WRLCK1; | |||
| 206 | //lock entire file | |||
| 207 | fl.l_whence = SEEK_SET0; | |||
| 208 | fl.l_start = 0; | |||
| 209 | fl.l_len = 0; | |||
| 210 | ||||
| 211 | while (true) { | |||
| 212 | int lock = fcntl(m_file, F_SETLK6, &fl); | |||
| 213 | if (lock == 0) | |||
| 214 | return true; | |||
| 215 | else if (std::chrono::steady_clock::now() > maxtime) | |||
| 216 | break; | |||
| 217 | if (errno(*__errno_location ()) != EAGAIN11 && errno(*__errno_location ()) != EACCES13 && errno(*__errno_location ()) != EINTR4) break; | |||
| 218 | usleep(uniform(random) * 1000); | |||
| 219 | } | |||
| 220 | if (!ignoreErrors) B2ERROR("Locking failed: " << strerror(errno))do { { LogVariableStream varStream; varStream << "Locking failed: " << strerror((*__errno_location ())); Belle2::LogSystem ::Instance().sendMessage(Belle2::LogMessage(Belle2::LogConfig ::c_Error, std::move(varStream), "framework", __PRETTY_FUNCTION__ , "framework/utilities/src/FileSystem.cc", 220, 0)); } } while (false); | |||
| 221 | return false; | |||
| 222 | } | |||
| 223 | ||||
| 224 | FileSystem::TemporaryFile::TemporaryFile(std::ios_base::openmode mode): std::fstream() | |||
| 225 | { | |||
| 226 | char* temporaryFileName = strdup((std::filesystem::temp_directory_path() / "basf2_XXXXXX").c_str()); | |||
| 227 | int fileDescriptor = mkstemp(temporaryFileName); | |||
| 228 | if (fileDescriptor == -1) { | |||
| 229 | B2ERROR("Cannot create temporary file: " << strerror(errno))do { { LogVariableStream varStream; varStream << "Cannot create temporary file: " << strerror((*__errno_location ())); Belle2::LogSystem ::Instance().sendMessage(Belle2::LogMessage(Belle2::LogConfig ::c_Error, std::move(varStream), "framework", __PRETTY_FUNCTION__ , "framework/utilities/src/FileSystem.cc", 229, 0)); } } while (false); | |||
| 230 | free(temporaryFileName); | |||
| 231 | return; | |||
| 232 | } | |||
| 233 | m_filename = std::string(temporaryFileName); | |||
| 234 | open(temporaryFileName, mode); | |||
| 235 | if (!is_open()) { | |||
| 236 | B2ERROR("Cannot open temporary file: " << strerror(errno))do { { LogVariableStream varStream; varStream << "Cannot open temporary file: " << strerror((*__errno_location ())); Belle2::LogSystem ::Instance().sendMessage(Belle2::LogMessage(Belle2::LogConfig ::c_Error, std::move(varStream), "framework", __PRETTY_FUNCTION__ , "framework/utilities/src/FileSystem.cc", 236, 0)); } } while (false); | |||
| 237 | } | |||
| 238 | free(temporaryFileName); | |||
| 239 | ::close(fileDescriptor); | |||
| 240 | } | |||
| 241 | ||||
| 242 | FileSystem::TemporaryFile::~TemporaryFile() | |||
| 243 | { | |||
| 244 | close(); | |||
| 245 | fs::remove(m_filename); | |||
| 246 | } |