111 ):
112 """Compares the concrete estimate to the truth and generates plots of the estimates, residuals, pulls and p-values.
113 Close indicates if the figure shall be closed after they are saved.
114
115 Parameters
116 ----------
117 truths : array_like(float)
118 Sample of the true values
119 estimates : array_like(float)
120 Corresponding estimations
121 variances : array_like(float), optional
122 Corresponding variance estimations
123 auxiliaries : Dict[name, array_like(float)], optional
124 Auxiliary variable to show distribution of residuals and pull as function
125 selected_plots : list(str), optional
126 List of analysis plots to be generated. All if not given.
127 Currently valid names are
128 truths, estimates, diag_profile, diag_scatter, residuals,
129 sigmas, pulls, p_values
130 """
131
132 if is_expert is None:
133 is_expert = self.default_is_expert
134
135 if which_plots is None:
136 which_plots = self.default_which_plots
137
138 quantity_name = self.quantity_name
139
140 axis_label = compose_axis_label(quantity_name, self.unit)
141
142 plot_name_prefix = self.plot_name_prefix
143 outlier_z_score = self.outlier_z_score
144
145 absolute = self.absolute
146
147
148 if absolute:
149 absolute_truths = truths.copy()
150 absolute_estimates = estimates.copy()
151
152 flip_sign_for = truths < 0
153 absolute_truths[flip_sign_for] = -truths[flip_sign_for]
154 absolute_estimates[flip_sign_for] = -estimates[flip_sign_for]
155
156 truths = absolute_truths
157 estimates = absolute_estimates
158
159 quantity_name = 'absolute ' + quantity_name
160
161 residuals = estimates - truths
162
163 if variances is not None:
164 sigmas = np.sqrt(variances)
165 pulls = np.divide(residuals, sigmas)
166 p_values = 1.0 - erf(np.abs(pulls))
167
168 plot_name = self.plot_name
169 if plot_name is None:
170 plot_name = self.default_plot_name
171
172 plot_name = formatter.format(plot_name,
173 quantity_name=quantity_name,
174 plot_name_prefix=plot_name_prefix,
175 plot_name_postfix=self.plot_name_postfix)
176
177 plot_title = self.plot_title
178 if plot_title is None:
179 plot_title = self.default_plot_title
180
181 plot_title = formatter.format(plot_title,
182 quantity_name=quantity_name,
183 plot_title_postfix=self.plot_title_postfix)
184
185
186
187
188 if "truths" in which_plots:
189
190 truths_hist_name = formatter.format(plot_name, subplot_name="truths")
191 truths_hist = ValidationPlot(truths_hist_name, self.referenceFileName)
192 truths_hist.hist(truths,
193 outlier_z_score=outlier_z_score,
194 is_expert=is_expert)
195 truths_hist.xlabel = axis_label
196 truths_hist.title = formatter.format(plot_title, subplot_title='True distribution')
197
198 self.plots['truths'] = truths_hist
199
200
201
202
203 if "estimates" in which_plots:
204
205 estimates_hist_name = formatter.format(plot_name, subplot_name="estimates")
206 estimates_hist = ValidationPlot(estimates_hist_name, self.referenceFileName)
207 estimates_hist.hist(estimates,
208 outlier_z_score=outlier_z_score,
209 is_expert=is_expert)
210 estimates_hist.xlabel = axis_label
211 estimates_hist.title = formatter.format(plot_title, subplot_title='Estimates distribution')
212
213 self.plots['estimates'] = estimates_hist
214
215
216
217 if "diag_scatter" in which_plots:
218
219 estimates_by_truths_scatter_name = formatter.format(plot_name, subplot_name="diag_scatter")
220 estimates_by_truths_scatter = ValidationPlot(estimates_by_truths_scatter_name, self.referenceFileName)
221 estimates_by_truths_scatter.scatter(truths,
222 estimates,
223 outlier_z_score=outlier_z_score,
224 is_expert=is_expert)
225 estimates_by_truths_scatter.xlabel = 'True ' + axis_label
226 estimates_by_truths_scatter.ylabel = 'Estimated ' + axis_label
227 estimates_by_truths_scatter.title = formatter.format(plot_title, subplot_title='Diagonal scatter plot')
228
229 self.plots['diag_scatter'] = estimates_by_truths_scatter
230
231 if "diag_profile" in which_plots:
232
233 estimates_by_truths_profile_name = formatter.format(plot_name, subplot_name="diag_profile")
234 estimates_by_truths_profile = ValidationPlot(estimates_by_truths_profile_name, self.referenceFileName)
235
236
237 estimates_by_truths_profile.profile(truths,
238 estimates - truths,
239 outlier_z_score=outlier_z_score,
240 gaus_z_score=4,
241 is_expert=is_expert)
242
243
244 hist = estimates_by_truths_profile.histograms[0]
245 GetBinContent = hist.GetBinContent
246 GetBinCenter = hist.GetBinCenter
247 SetBinContent = hist.SetBinContent
248 for i_bin in range(hist.GetNbinsX() + 2):
249 residual = GetBinContent(i_bin)
250 truth = GetBinCenter(i_bin)
251 if residual != 0:
252 SetBinContent(i_bin, residual + truth)
253
254
255 estimates_by_truths_profile.histograms[0].SetMaximum()
256 estimates_by_truths_profile.histograms[0].SetMinimum()
257
258 estimates_by_truths_profile.xlabel = 'True ' + axis_label
259 estimates_by_truths_profile.ylabel = 'Estimated ' + axis_label
260
261 estimates_by_truths_profile.title = formatter.format(plot_title, subplot_title='Diagonal profile')
262 estimates_by_truths_profile.fit_diag()
263
264 self.plots['diag_profile'] = estimates_by_truths_profile
265
266
267
268 if "residuals" in which_plots:
269
270 residuals_hist_name = formatter.format(plot_name, subplot_name="residuals")
271 residuals_hist = ValidationPlot(residuals_hist_name, self.referenceFileName)
272 residuals_hist.hist(residuals,
273 outlier_z_score=outlier_z_score,
274 is_expert=is_expert)
275 residuals_hist.xlabel = compose_axis_label("#Delta " + quantity_name + " (estimate - truth)", self.unit)
276 residuals_hist.title = formatter.format(plot_title, subplot_title='Residual distribution')
277
278 self.plots['residuals'] = residuals_hist
279
280
281
282 if variances is not None and "sigmas" in which_plots:
283
284
285 sigmas_hist_name = formatter.format(plot_name, subplot_name="sigmas")
286 sigmas_hist = ValidationPlot(sigmas_hist_name, self.referenceFileName)
287 sigmas_hist.hist(sigmas,
288 lower_bound=0,
289 outlier_z_score=outlier_z_score,
290 is_expert=is_expert)
291 sigmas_hist.xlabel = compose_axis_label("#sigma (" + quantity_name + ')', self.unit)
292 sigmas_hist.title = formatter.format(plot_title, subplot_title='Estimated variance distribution')
293
294 self.plots['sigmas'] = sigmas_hist
295
296
297
298 if variances is not None and "pulls" in which_plots:
299
300
301 pulls_hist_name = formatter.format(plot_name, subplot_name="pulls")
302 pulls_hist = ValidationPlot(pulls_hist_name, self.referenceFileName)
303 pulls_hist.hist(pulls, outlier_z_score=outlier_z_score, is_expert=is_expert)
304 pulls_hist.xlabel = "pull (" + quantity_name + ")"
305 pulls_hist.title = formatter.format(plot_title, subplot_title='Pull distribution')
306 pulls_hist.fit_gaus(z_score=1)
307
308 self.plots['pulls'] = pulls_hist
309
310
311
312 if variances is not None and "p_values" in which_plots:
313
314
315 p_values_hist_name = formatter.format(plot_name, subplot_name="p-values")
316 p_values_hist = ValidationPlot(p_values_hist_name, self.referenceFileName)
317 p_values_hist.hist(p_values, lower_bound=0, upper_bound=1, is_expert=is_expert)
318 p_values_hist.xlabel = "p-value (" + quantity_name + ")"
319 p_values_hist.title = formatter.format(plot_title, subplot_title='P-value distribution')
320 p_values_hist.fit_const()
321
322 self.plots['p_values'] = p_values_hist
323
324
325
326 for aux_name, aux_values in auxiliaries.items():
327 if "aux_residual_hist2d" in which_plots or "aux" in which_plots:
328
329 aux_residuals_hist2d_name = formatter.format(plot_name,
330 subplot_name=f"residuals over {aux_name}")
331 aux_residuals_hist2d = ValidationPlot(aux_residuals_hist2d_name, self.referenceFileName)
332 aux_residuals_hist2d.hist2d(aux_values,
333 residuals,
334 outlier_z_score=outlier_z_score,
335 allow_discrete=True,
336 is_expert=is_expert)
337 aux_residuals_hist2d.xlabel = compose_axis_label(aux_name)
338 aux_residuals_hist2d.ylabel = compose_axis_label("#Delta " + quantity_name + " (estimate - truth)", self.unit)
339 aux_residuals_hist2d.title = formatter.format(plot_title,
340 subplot_title=f'Residual distribution over {aux_name}')
341
342 self.plots['aux_residuals_hist2d_' + aux_name] = aux_residuals_hist2d
343
344 if "aux_residual_profile" in which_plots or "aux" in which_plots:
345
346 aux_residuals_profile_name = formatter.format(plot_name,
347 subplot_name=f"residuals profile over {aux_name}")
348 aux_residuals_profile = ValidationPlot(aux_residuals_profile_name, self.referenceFileName)
349 aux_residuals_profile.profile(aux_values,
350 residuals,
351 outlier_z_score=outlier_z_score,
352 gaus_z_score=1.5,
353 allow_discrete=True,
354 is_expert=is_expert,
355 )
356 aux_residuals_profile.xlabel = compose_axis_label(aux_name)
357 aux_residuals_profile.ylabel = compose_axis_label("#Delta " + quantity_name + " (estimate - truth)", self.unit)
358 aux_residuals_profile.title = formatter.format(plot_title,
359 subplot_title=f'Residual profile over {aux_name}')
360
361 self.plots['aux_residuals_profile_' + aux_name] = aux_residuals_profile
362
363 if variances is not None and ("aux_pull_hist2d" in which_plots or "aux" in which_plots):
364
365 aux_pulls_hist2d_name = formatter.format(plot_name,
366 subplot_name=f"pulls over {aux_name}")
367 aux_pulls_hist2d = ValidationPlot(aux_pulls_hist2d_name, self.referenceFileName)
368 aux_pulls_hist2d.hist2d(aux_values,
369 pulls,
370 outlier_z_score=outlier_z_score,
371 allow_discrete=True,
372 is_expert=is_expert)
373 aux_pulls_hist2d.xlabel = compose_axis_label(aux_name)
374 aux_pulls_hist2d.ylabel = "pull (" + quantity_name + ")"
375 aux_pulls_hist2d.title = formatter.format(plot_title,
376 subplot_title=f'Pull scatter over {aux_name}')
377
378 self.plots['aux_pulls_hist2d_' + aux_name] = aux_pulls_hist2d
379
380 if variances is not None and ("aux_pull_profile" in which_plots or "aux" in which_plots):
381
382 aux_pulls_profile_name = formatter.format(plot_name,
383 subplot_name=f"pull profile over {aux_name}")
384 aux_pulls_profile = ValidationPlot(aux_pulls_profile_name, self.referenceFileName)
385 aux_pulls_profile.profile(aux_values,
386 pulls,
387 outlier_z_score=outlier_z_score,
388 gaus_z_score=1.5,
389 allow_discrete=True,
390 is_expert=is_expert)
391 aux_pulls_profile.xlabel = compose_axis_label(aux_name)
392 aux_pulls_profile.ylabel = "pull (" + quantity_name + ")"
393 aux_pulls_profile.title = formatter.format(plot_title,
394 subplot_title=f'Pull profile over {aux_name}')
395
396 self.plots['aux_pulls_profile_' + aux_name] = aux_pulls_profile
397
398
399 self.contact = self.contact
400
1.9.6