15 import tensorflow
as tf
21 """ Using class to stop training early if it's not getting better"""
32 Check if validation result is better than the best validation result.
33 Decide if training should be continued.
48 def get_model(number_of_features, number_of_spectators, number_of_events, training_fraction, parameters):
49 """Building Graph inside tensorflow"""
51 gpus = tf.config.list_physical_devices(
'GPU')
54 tf.config.experimental.set_memory_growth(gpu,
True)
56 class my_model(tf.Module):
61 self.optimizer = tf.optimizers.Adam(0.01)
62 self.pre_optimizer = tf.optimizers.Adam(0.01)
64 def create_layer_variables(shape, name, activation_function):
65 weights = tf.Variable(
66 tf.random.truncated_normal(shape, stddev=1.0 / np.sqrt(float(shape[0]))),
67 name=f
'{name}_weights')
68 biases = tf.Variable(tf.zeros(shape=[shape[1]]), name=f
'{name}_biases')
69 return weights, biases, activation_function
71 n = int(number_of_features / 6)
72 self.number_of_track_pairs = int(n * (n - 1) / 2)
73 self.number_of_features_per_relation = 1
74 self.parameters = parameters
77 self.rel_net_vars = []
78 shape = [12, 50, 50, 1]
79 for i
in range(len(shape) - 1):
81 if i == len(shape) - 2:
83 self.rel_net_vars.append(create_layer_variables(shape[i:i + 2], f
'relation_{i}', unit))
85 self.inference_vars = []
87 n_features_in = (self.number_of_track_pairs
if self.parameters[
'use_relations']
else number_of_features)
89 if self.parameters[
'use_feed_forward']:
90 self.inference_vars.append(create_layer_variables([n_features_in, 50],
'inf_hidden_1', tf.nn.relu))
91 self.inference_vars.append(create_layer_variables([50, 50],
'inf_hidden_2', tf.nn.relu))
92 self.inference_vars.append(create_layer_variables([50, 1],
'inf_activation', tf.nn.sigmoid))
94 self.inference_vars.append(create_layer_variables([n_features_in, 1],
'inf_activation', tf.nn.sigmoid))
96 def dense(self, x, W, b, activation_function):
97 return activation_function(tf.matmul(x, W) + b)
99 @tf.function(input_signature=[tf.TensorSpec(shape=[None, number_of_features], dtype=tf.float32)])
100 def prepare_x(self, x):
101 """ prepare the features for use in the relation net """
102 if not self.parameters[
'use_relations']:
106 for i
in range(int(number_of_features / 6)):
107 tracks.append(tf.slice(x, [0, i * 6], [-1, 6]))
111 for feature_number
in range(self.number_of_features_per_relation):
113 for counter, track1
in enumerate(tracks):
114 for track2
in tracks[counter + 1:]:
115 relations.append(self.relation_net(tf.concat([track1, track2], 1), self.rel_net_vars))
120 return tf.concat(relations, 1)
124 @tf.function(input_signature=[tf.TensorSpec(shape=[None, 12], dtype=tf.float32)])
125 def pre_train(self, z):
126 """build net for pre-training with the same shared variables """
127 if not self.parameters[
'use_relations']:
130 pre_training_relations = []
131 for feature_number
in range(self.number_of_features_per_relation):
132 if self.parameters[
'pre_training_epochs'] > 0:
133 pre_training_relations.append(self.relation_net(z, self.rel_net_vars))
135 if pre_training_relations:
136 return tf.concat(pre_training_relations, 1)
140 def relation_net(self, x, variables):
141 """Build one relation net between 2 object using pre-build variables"""
142 for layer
in variables:
143 x = self.dense(x, *layer)
146 @tf.function(input_signature=[tf.TensorSpec(shape=[None, number_of_features], dtype=tf.float32)])
147 def __call__(self, x):
148 x = self.prepare_x(x)
149 for variables
in self.inference_vars:
150 x = self.dense(x, *variables)
154 def loss(self, predicted_y, target_y, w):
156 diff_from_truth = tf.where(target_y == 1., predicted_y, 1. - predicted_y)
157 return - tf.reduce_sum(w * tf.math.log(diff_from_truth + epsilon)) / tf.reduce_sum(w)
159 state =
State(model=my_model())
163 def begin_fit(state, Xtest, Stest, ytest, wtest):
164 """Saves the training validation set for monitoring."""
172 def partial_fit(state, X, S, y, w, epoch):
173 """Pass received data to tensorflow session"""
175 def variable_is_trainable(var, parameters, pre_training=False):
177 if not parameters[
'use_relations']
and 'relation_' in var.name:
179 if pre_training
and 'inf_' in var.name:
185 if state.model.parameters[
'pre_training_epochs'] > epoch:
186 pre_trainable_variables = [
187 x
for x
in state.model.trainable_variables
if variable_is_trainable(
188 x, state.model.parameters,
True)]
189 with tf.GradientTape()
as tape:
190 avg_cost = state.model.loss(state.model.pre_train(S), y, w)
191 grads = tape.gradient(avg_cost, pre_trainable_variables)
193 state.model.pre_optimizer.apply_gradients(zip(grads, pre_trainable_variables))
195 if epoch % 1000 == 0:
196 print(
"Pre-Training: Epoch:",
'%04d' % (epoch),
"cost=",
"{:.9f}".format(avg_cost))
200 trainable_variables = [
201 x
for x
in state.model.trainable_variables
if variable_is_trainable(
202 x, state.model.parameters,
False)]
203 with tf.GradientTape()
as tape:
204 avg_cost = state.model.loss(state.model(X), y, w)
205 grads = tape.gradient(avg_cost, trainable_variables)
207 state.model.optimizer.apply_gradients(zip(grads, trainable_variables))
209 if epoch % 1000 == 0:
210 print(
"Epoch:",
'%04d' % (epoch),
"cost=",
"{:.9f}".format(avg_cost))
211 return EARLY_STOPPER.check(avg_cost)
215 if __name__ ==
"__main__":
218 from root_pandas
import to_root
223 import basf2_mva_util
234 number_total_lines = 5
236 for i
in range(number_total_lines):
237 variables += [
'px_' + str(i),
'py_' + str(i),
'pz_' + str(i),
'dx_' + str(i),
'dy_' + str(i),
241 spectators = [
'Spx1',
'Spy1',
'Spz1',
'Sdx1',
'Sdy1',
'Sdz1',
'Spx2',
'Spy2',
'Spz2',
'Sdx2',
'Sdy2',
'Sdz2']
243 number_of_events = 100000
245 def build_signal_event():
246 """Building two lines which are hitting each other"""
247 p_vec1, p_vec2 = np.random.normal(size=3), np.random.normal(size=3)
248 v_cross = np.random.normal(size=3)
249 epsilon1, epsilon2 = np.random.rand() * 2 - 1, np.random.rand() * 2 - 1
250 v_vec1 = v_cross + (p_vec1 * epsilon1)
251 v_vec2 = v_cross + (p_vec2 * epsilon2)
252 return np.concatenate([p_vec1, v_vec1]), np.concatenate([p_vec2, v_vec2])
255 with tempfile.TemporaryDirectory()
as path:
256 for filename
in [
'train.root',
'test.root']:
257 print(
'Building ' + filename)
259 data = np.random.normal(size=[number_of_events, number_total_lines * 6 + 12])
260 target = np.zeros([number_of_events], dtype=bool)
264 for index, sample
in enumerate(data):
265 if np.random.rand() > 0.5:
267 i1, i2 = int(np.random.rand() * number_total_lines), int(np.random.rand() * (number_total_lines - 1))
268 i2 = (i1 + i2) % number_total_lines
269 track1, track2 = build_signal_event()
270 data[index, i1 * 6:(i1 + 1) * 6] = track1
271 data[index, i2 * 6:(i2 + 1) * 6] = track2
272 data[index, number_total_lines * 6:] = np.append(track1, track2)
276 for i, name
in enumerate(variables + spectators):
277 dic.update({name: data[:, i]})
278 dic.update({
'isSignal': target})
280 df = pandas.DataFrame(dic, dtype=np.float32)
281 to_root(df, os.path.join(path, filename), key=
'variables')
286 general_options = basf2_mva.GeneralOptions()
287 general_options.m_datafiles = basf2_mva.vector(os.path.join(path,
'train.root'))
288 general_options.m_treename =
"variables"
289 general_options.m_variables = basf2_mva.vector(*variables)
290 general_options.m_target_variable =
"isSignal"
291 general_options.m_spectators = basf2_mva.vector(*spectators)
293 specific_options = basf2_mva.PythonOptions()
294 specific_options.m_framework =
"tensorflow"
295 specific_options.m_steering_file =
'mva/examples/tensorflow/relations.py'
296 specific_options.m_nIterations = 100
297 specific_options.m_mini_batch_size = 100
299 print(
'Train relational net with pre-training')
300 general_options.m_identifier = os.path.join(path,
'relation_2.xml')
301 specific_options.m_config = json.dumps({
'use_relations':
True,
'use_feed_forward':
False,
'pre_training_epochs': 3000})
302 basf2_mva.teacher(general_options, specific_options)
304 print(
'Train feed forward net')
305 general_options.m_identifier = os.path.join(path,
'feed_forward.xml')
306 specific_options.m_config = json.dumps({
'use_relations':
False,
'use_feed_forward':
True,
'pre_training_epochs': 0})
307 basf2_mva.teacher(general_options, specific_options)
309 print(
'Train relational net')
310 general_options.m_identifier = os.path.join(path,
'relation.xml')
311 specific_options.m_config = json.dumps({
'use_relations':
True,
'use_feed_forward':
True,
'pre_training_epochs': 0})
312 basf2_mva.teacher(general_options, specific_options)
320 test_data = basf2_mva.vector(os.path.join(path,
'test.root'))
321 print(
'Apply feed forward net')
322 p1, t1 = method1.apply_expert(test_data, general_options.m_treename)
323 print(
'Apply relational net')
324 p2, t2 = method2.apply_expert(test_data, general_options.m_treename)
325 print(
'Apply special relational net')
326 p3, t3 = method3.apply_expert(test_data, general_options.m_treename)
def calculate_auc_efficiency_vs_background_retention(p, t, w=None)
best_result
saves best training result
counter
counts how many times training is not getting better