Belle II Software light-2406-ragdoll
Weightfile.cc
1/**************************************************************************
2 * basf2 (Belle II Analysis Software Framework) *
3 * Author: The Belle II Collaboration *
4 * *
5 * See git log for contributors and copyright holders. *
6 * This file is licensed under LGPL-3.0, see LICENSE.md. *
7 **************************************************************************/
8
9#include <mva/interface/Weightfile.h>
10
11#include <mva/dataobjects/DatabaseRepresentationOfWeightfile.h>
12#include <framework/database/Database.h>
13#include <framework/database/DBImportArray.h>
14
15#include <boost/archive/iterators/base64_from_binary.hpp>
16#include <boost/archive/iterators/binary_from_base64.hpp>
17#include <boost/archive/iterators/transform_width.hpp>
18
19#include <boost/property_tree/xml_parser.hpp>
20#include <boost/algorithm/string/predicate.hpp>
21#include <boost/algorithm/string.hpp>
22#include <boost/algorithm/string/replace.hpp>
23#include <boost/regex.hpp>
24
25#include <TFile.h>
26
27#include <sstream>
28#include <filesystem>
29
30namespace fs = std::filesystem;
31
32namespace Belle2 {
37 namespace MVA {
38
39 std::string makeSaveForDatabase(std::string str)
40 {
41 std::map<std::string, std::string> replace {
42 {" ", "__sp"},
43 };
44 for (auto& pair : replace) {
45 boost::replace_all(str, pair.first, pair.second);
46 }
47 //const static boost::regex blackList("[^a-zA-Z0-9_]");
48 //return boost::regex_replace(str, blackList, "");
49 return str;
50 }
51
53 {
54 for (auto& filename : m_filenames) {
55 if (fs::exists(filename)) {
57 fs::remove_all(filename);
58 }
59 }
60 }
61
62 void Weightfile::addOptions(const Options& options)
63 {
64 options.save(m_pt);
65 }
66
67 void Weightfile::getOptions(Options& options) const
68 {
69 options.load(m_pt);
70 }
71
72 void Weightfile::addFeatureImportance(const std::map<std::string, float>& importance)
73 {
74 m_pt.put("number_of_importance_vars", importance.size());
75 unsigned int i = 0;
76 for (auto& pair : importance) {
77 m_pt.put(std::string("importance_key") + std::to_string(i), pair.first);
78 m_pt.put(std::string("importance_value") + std::to_string(i), pair.second);
79 ++i;
80 }
81 }
82
83 std::map<std::string, float> Weightfile::getFeatureImportance() const
84 {
85 std::map<std::string, float> importance;
86 unsigned int numberOfImportanceVars = m_pt.get<unsigned int>("number_of_importance_vars", 0);
87 for (unsigned int i = 0; i < numberOfImportanceVars; ++i) {
88 auto key = m_pt.get<std::string>(std::string("importance_key") + std::to_string(i));
89 auto value = m_pt.get<float>(std::string("importance_value") + std::to_string(i));
90 importance[key] = value;
91 }
92 return importance;
93 }
94
95 void Weightfile::addSignalFraction(float signal_fraction)
96 {
97 m_pt.put("signal_fraction", signal_fraction);
98 }
99
101 {
102 return m_pt.get<float>("signal_fraction");
103 }
104
105 std::string Weightfile::generateFileName(const std::string& suffix)
106 {
107 char* directory_template = strdup((fs::temp_directory_path() / "Basf2MVA.XXXXXX").c_str());
108 auto directory = mkdtemp(directory_template);
109 std::string tmpfile = std::string(directory) + std::string("/weightfile") + suffix;
110 m_filenames.emplace_back(directory);
111 free(directory_template);
112 return tmpfile;
113 }
114
115 void Weightfile::addFile(const std::string& identifier, const std::string& custom_weightfile)
116 {
117 // TODO Test if file is valid
118 std::ifstream in(custom_weightfile, std::ios::in | std::ios::binary);
119 addStream(identifier, in);
120 in.close();
121 }
122
123 void Weightfile::addStream(const std::string& identifier, std::istream& in)
124 {
125 using base64_t = boost::archive::iterators::base64_from_binary <
126 boost::archive::iterators::transform_width<std::string::const_iterator, 6, 8 >>;
127
128 std::string contents;
129 in.seekg(0, std::ios::end);
130 contents.resize(in.tellg());
131 in.seekg(0, std::ios::beg);
132 in.read(&contents[0], contents.size());
133 std::string enc(base64_t(contents.begin()), base64_t(contents.end()));
134
135 m_pt.put(identifier, enc);
136 }
137
138 void Weightfile::getFile(const std::string& identifier, const std::string& custom_weightfile)
139 {
140 std::ofstream out(custom_weightfile, std::ios::out | std::ios::binary);
141 out << getStream(identifier);
142 }
143
144 std::string Weightfile::getStream(const std::string& identifier) const
145 {
146 using binary_t = boost::archive::iterators::transform_width <
147 boost::archive::iterators::binary_from_base64<std::string::const_iterator>, 8, 6 >;
148
149 auto contents = m_pt.get<std::string>(identifier);
150 std::string dec(binary_t(contents.begin()), binary_t(contents.end()));
151 return dec;
152 }
153
154 void Weightfile::save(Weightfile& weightfile, const std::string& filename, const Belle2::IntervalOfValidity& iov)
155 {
156 if (boost::ends_with(filename, ".root")) {
157 return saveToROOTFile(weightfile, filename);
158 } else if (boost::ends_with(filename, ".xml")) {
159 return saveToXMLFile(weightfile, filename);
160 } else {
161 return saveToDatabase(weightfile, filename, iov);
162 }
163 }
164
165 void Weightfile::saveToROOTFile(Weightfile& weightfile, const std::string& filename)
166 {
167 std::stringstream ss;
168 Weightfile::saveToStream(weightfile, ss);
169 DatabaseRepresentationOfWeightfile database_representation_of_weightfile;
170 database_representation_of_weightfile.m_data = ss.str();
171 TFile file(filename.c_str(), "RECREATE");
172 file.WriteObject(&database_representation_of_weightfile, "Weightfile");
173 }
174
175 void Weightfile::saveToXMLFile(Weightfile& weightfile, const std::string& filename)
176 {
177#if BOOST_VERSION < 105600
178 boost::property_tree::xml_writer_settings<char> settings('\t', 1);
179#else
180 boost::property_tree::xml_writer_settings<std::string> settings('\t', 1);
181#endif
182 boost::property_tree::xml_parser::write_xml(filename, weightfile.m_pt, std::locale(), settings);
183 }
184
185 void Weightfile::saveToStream(Weightfile& weightfile, std::ostream& stream)
186 {
187#if BOOST_VERSION < 105600
188 boost::property_tree::xml_writer_settings<char> settings('\t', 1);
189#else
190 boost::property_tree::xml_writer_settings<std::string> settings('\t', 1);
191#endif
192 boost::property_tree::xml_parser::write_xml(stream, weightfile.m_pt, settings);
193 }
194
195 Weightfile Weightfile::load(const std::string& filename, const Belle2::EventMetaData& emd)
196 {
197 if (boost::ends_with(filename, ".root")) {
198 return loadFromROOTFile(filename);
199 } else if (boost::ends_with(filename, ".xml")) {
200 return loadFromXMLFile(filename);
201 } else {
202 return loadFromDatabase(filename, emd);
203 }
204 }
205
206 Weightfile Weightfile::loadFromFile(const std::string& filename)
207 {
208 if (boost::ends_with(filename, ".root")) {
209 return loadFromROOTFile(filename);
210 } else if (boost::ends_with(filename, ".xml")) {
211 return loadFromXMLFile(filename);
212 } else {
213 throw std::runtime_error("Cannot load file " + filename + " because file extension is not supported");
214 }
215 }
216
217 Weightfile Weightfile::loadFromROOTFile(const std::string& filename)
218 {
219
220 if (not fs::exists(filename)) {
221 throw std::runtime_error("Given filename does not exist: " + filename);
222 }
223
224 TFile file(filename.c_str(), "READ");
225 if (file.IsZombie() or not file.IsOpen()) {
226 throw std::runtime_error("Error during open of ROOT file named " + filename);
227 }
228
229 DatabaseRepresentationOfWeightfile* database_representation_of_weightfile = nullptr;
230 file.GetObject("Weightfile", database_representation_of_weightfile);
231
232 if (database_representation_of_weightfile == nullptr) {
233 throw std::runtime_error("The provided ROOT file " + filename + " does not contain a valid MVA weightfile.");
234 }
235 std::stringstream ss(database_representation_of_weightfile->m_data);
236 delete database_representation_of_weightfile;
238 }
239
240 Weightfile Weightfile::loadFromXMLFile(const std::string& filename)
241 {
242 if (not fs::exists(filename)) {
243 throw std::runtime_error("Given filename does not exist: " + filename);
244 }
245
246 Weightfile weightfile;
247 boost::property_tree::xml_parser::read_xml(filename, weightfile.m_pt);
248 return weightfile;
249 }
250
252 {
253 Weightfile weightfile;
254 boost::property_tree::xml_parser::read_xml(stream, weightfile.m_pt);
255 return weightfile;
256 }
257
258 void Weightfile::saveToDatabase(Weightfile& weightfile, const std::string& identifier, const Belle2::IntervalOfValidity& iov)
259 {
260 std::stringstream ss;
261 Weightfile::saveToStream(weightfile, ss);
262 DatabaseRepresentationOfWeightfile database_representation_of_weightfile;
263 database_representation_of_weightfile.m_data = ss.str();
264 Belle2::Database::Instance().storeData(makeSaveForDatabase(identifier), &database_representation_of_weightfile, iov);
265 }
266
267 void Weightfile::saveArrayToDatabase(const std::vector<Weightfile>& weightfiles, const std::string& identifier,
269 {
270 DBImportArray<DatabaseRepresentationOfWeightfile> dbArray(makeSaveForDatabase(identifier));
271
272 for (auto weightfile : weightfiles) {
273 std::stringstream ss;
274 Weightfile::saveToStream(weightfile, ss);
275 dbArray.appendNew(ss.str());
276 }
277
278 dbArray.import(iov);
279 }
280
281 Weightfile Weightfile::loadFromDatabase(const std::string& identifier, const Belle2::EventMetaData& emd)
282 {
283 auto pair = Belle2::Database::Instance().getData(emd, makeSaveForDatabase(identifier));
284
285 if (pair.first == 0) {
286 throw std::runtime_error("Given identifier cannot be loaded from the database: " + identifier);
287 }
288
289 DatabaseRepresentationOfWeightfile database_representation_of_weightfile = *static_cast<DatabaseRepresentationOfWeightfile*>
290 (pair.first);
291 std::stringstream ss(database_representation_of_weightfile.m_data);
293 }
294
295 }
297}
Class for importing array of objects to the database.
Definition: DBImportArray.h:25
T * appendNew()
Construct a new T object at the end of the array.
Definition: DBImportArray.h:67
bool import(const IntervalOfValidity &iov)
Import the object to database.
Definition: DBImportBase.cc:36
Database representation of a Weightfile object.
Store event, run, and experiment numbers.
Definition: EventMetaData.h:33
A class that describes the interval of experiments/runs for which an object in the database is valid.
Abstract base class of all Options given to the MVA interface.
Definition: Options.h:34
virtual void load(const boost::property_tree::ptree &pt)=0
Load mechanism (used by Weightfile) to load Options from a xml tree.
virtual void save(boost::property_tree::ptree &pt) const =0
Save mechanism (used by Weightfile) to store Options in a xml tree.
The Weightfile class serializes all information about a training into an xml tree.
Definition: Weightfile.h:38
void addStream(const std::string &identifier, std::istream &in)
Add a stream to our weightfile.
Definition: Weightfile.cc:123
void addFile(const std::string &identifier, const std::string &custom_weightfile)
Add a file (mostly a weightfile from a MVA library) to our Weightfile.
Definition: Weightfile.cc:115
std::map< std::string, float > getFeatureImportance() const
Get feature importance.
Definition: Weightfile.cc:83
static Weightfile loadFromXMLFile(const std::string &filename)
Static function which loads a Weightfile from a XML file.
Definition: Weightfile.cc:240
static void save(Weightfile &weightfile, const std::string &filename, const Belle2::IntervalOfValidity &iov=Belle2::IntervalOfValidity(0, 0, -1, -1))
Static function which saves a Weightfile to a file.
Definition: Weightfile.cc:154
~Weightfile()
Destructor (removes temporary files associated with this weightfiles)
Definition: Weightfile.cc:52
bool m_remove_temporary_directories
remove all temporary directories in the destructor of this class
Definition: Weightfile.h:292
static void saveToXMLFile(Weightfile &weightfile, const std::string &filename)
Static function which saves a Weightfile to a XML file.
Definition: Weightfile.cc:175
static Weightfile loadFromStream(std::istream &stream)
Static function which deserializes a Weightfile from a stream.
Definition: Weightfile.cc:251
boost::property_tree::ptree m_pt
xml tree containing all the saved information of this weightfile
Definition: Weightfile.h:290
void addOptions(const Options &options)
Add an Option object to the xml tree.
Definition: Weightfile.cc:62
static Weightfile loadFromROOTFile(const std::string &filename)
Static function which loads a Weightfile from a ROOT file.
Definition: Weightfile.cc:217
void getOptions(Options &options) const
Fills an Option object from the xml tree.
Definition: Weightfile.cc:67
static Weightfile load(const std::string &filename, const Belle2::EventMetaData &emd=Belle2::EventMetaData(0, 0, 0))
Static function which loads a Weightfile from a file or from the database.
Definition: Weightfile.cc:195
static Weightfile loadFromDatabase(const std::string &identifier, const Belle2::EventMetaData &emd=Belle2::EventMetaData(0, 0, 0))
Static function which loads a Weightfile from the basf2 condition database.
Definition: Weightfile.cc:281
static void saveToStream(Weightfile &weightfile, std::ostream &stream)
Static function which serializes a Weightfile to a stream.
Definition: Weightfile.cc:185
static Weightfile loadFromFile(const std::string &filename)
Static function which loads a Weightfile from a file.
Definition: Weightfile.cc:206
void addSignalFraction(float signal_fraction)
Saves the signal fraction in the xml tree.
Definition: Weightfile.cc:95
std::vector< std::string > m_filenames
generated temporary filenames, which will be removed in the destructor of this class
Definition: Weightfile.h:291
void addFeatureImportance(const std::map< std::string, float > &importance)
Add variable importance.
Definition: Weightfile.cc:72
static void saveToROOTFile(Weightfile &weightfile, const std::string &filename)
Static function which saves a Weightfile to a ROOT file.
Definition: Weightfile.cc:165
float getSignalFraction() const
Loads the signal fraction frm the xml tree.
Definition: Weightfile.cc:100
static void saveArrayToDatabase(const std::vector< Weightfile > &weightfiles, const std::string &identifier, const Belle2::IntervalOfValidity &iov=Belle2::IntervalOfValidity(0, 0, -1, -1))
Static function which saves an array of Weightfile objects in the basf2 condition database.
Definition: Weightfile.cc:267
std::string generateFileName(const std::string &suffix="")
Returns a temporary filename with the given suffix.
Definition: Weightfile.cc:105
std::string getStream(const std::string &identifier) const
Returns the content of a stored stream as string.
Definition: Weightfile.cc:144
static void saveToDatabase(Weightfile &weightfile, const std::string &identifier, const Belle2::IntervalOfValidity &iov=Belle2::IntervalOfValidity(0, 0, -1, -1))
Static function which saves a Weightfile in the basf2 condition database.
Definition: Weightfile.cc:258
void getFile(const std::string &identifier, const std::string &custom_weightfile)
Creates a file from our weightfile (mostly this will be a weightfile of an MVA library)
Definition: Weightfile.cc:138
std::pair< TObject *, IntervalOfValidity > getData(const EventMetaData &event, const std::string &name)
Request an object from the database.
Definition: Database.cc:72
static Database & Instance()
Instance of a singleton Database.
Definition: Database.cc:42
bool storeData(const std::string &name, TObject *object, const IntervalOfValidity &iov)
Store an object in the database.
Definition: Database.cc:141
Abstract base class for different kinds of events.
Definition: ClusterUtils.h:24