11""" Compare ROOT objects and perform e.g. chi2 tests.
12A small command line interface for testing/debugging purposes is included.
13Run `python3 validationcomparison.py --help` for more information. """
16from abc
import ABC, abstractmethod
20from typing
import Optional, Any
26from metaoptions
import MetaOptionParser
38class ComparisonFailed(Exception):
40 The comparison failed for some reason. For example
41 because ROOT was not able to compute the Chi^2 properly
45class ObjectsNotSupported(Exception):
47 The type and/or combination of provided ROOT objects
48 is not supported for comparison
52class DifferingBinCount(Exception):
54 The two ROOT objects provided have a different bin count
55 and therefore, cannot be compared using the Chi2 test
59class TooFewBins(Exception):
61 Not sufficient bins to perform the Chi^2 test
71 object_1, object_2, mop: Optional[MetaOptionParser] =
None,
73 """ Uses the metaoptions to determine which comparison algorithm is used
74 and initializes the corresponding subclass of :class:`ComparisonBase` that
75 implements the actual comparison and holds the results.
76 @param object_1 ROOT TObject
77 @param object_2 ROOT TObject
78 @param mop Metaoption parser
81 mop = MetaOptionParser()
82 if mop.has_option(
"kolmogorov"):
83 tester: Any = KolmogorovTest
84 elif mop.has_option(
"andersondarling"):
85 tester = AndersonDarlingTest
89 test = tester(object_1, object_2, mop=mop)
99class ComparisonBase(ABC):
101 Base class for all comparison implementations.
105 1. Initialize the class together with two ROOT objects of different
106 revisions (that are to be compared) and the metaoptions (given in the
107 corresponding validation (steering) file), that determine how to compare
110 2. The Comparison class saves the ROOT objects and the metaoptions
111 internally, but does not compute anything yet
113 3. If :meth:`ensure_compute` is called, or any property is accessed that
114 depends on computation, the internal implementation :meth:`_compute`
115 (to be implemented in the subclass) is called.
117 4. :meth:`_compute` ensures that all values, like chi2, p-value etc. are
120 5. Two properties :meth:`comparison_result` (pass/warning/error) and
121 :meth:`comparison_result_long` (longer description of the comparison result)
122 allow to access the results.
129 mop: Optional[MetaOptionParser] =
None,
133 Initialize ComparisonBase class
137 :param mop: MetaOptionParser
138 :param debug (bool): Debug mode enabled?
141 self.object_a = object_a
144 self.object_b = object_b
148 mop = MetaOptionParser()
155 self.computed =
False
158 self._comparison_result =
"not_compared"
161 self._comparison_result_long =
""
163 def ensure_compute(self):
165 Ensure all required quantities get computed and are cached inside the
171 if self.mop.has_option(
"nocompare"):
173 self._comparison_result_long =
"Testing is disabled for this plot"
176 fail_message =
"Comparison failed: "
181 except ObjectsNotSupported
as e:
182 self._comparison_result_long = fail_message + str(e)
183 except DifferingBinCount
as e:
184 self._comparison_result =
"error"
185 self._comparison_result_long = fail_message + str(e)
186 except TooFewBins
as e:
187 self._comparison_result_long = fail_message + str(e)
188 except ComparisonFailed
as e:
189 self._comparison_result =
"error"
190 self._comparison_result_long = fail_message + str(e)
191 except Exception
as e:
192 self._comparison_result =
"error"
193 self._comparison_result_long = (
194 "Unknown error occurred. Please "
195 "submit a bug report. " + str(e)
200 self._comparison_result_long = self._get_comparison_result_long()
201 self._comparison_result = self._get_comparison_result()
206 def _get_comparison_result(self) -> str:
207 """ Used to format the value of :attr:`_comparison_result`. """
210 def _get_comparison_result_long(self) -> str:
211 """ Used to format the value of :attr:`_comparison_result_long`. """
214 def comparison_result(self):
215 """ Comparison result, i.e. pass/warning/error """
216 self.ensure_compute()
217 return self._comparison_result
220 def comparison_result_long(self):
221 """ Longer description of the comparison result """
222 self.ensure_compute()
223 return self._comparison_result_long
227 """ This method performs the actual computations. """
229 def can_compare(self):
231 @return: True if the two objects can be compared, False otherwise
233 return self._has_correct_types()
and self._has_compatible_bins()
235 def _has_correct_types(self) -> bool:
237 @return: True if the two objects have a) a type supported for
238 comparison and b) can be compared with each other
240 if self.object_a
is None or self.object_b
is None:
245 supported_types = [
"TProfile",
"TH1D",
"TH1F",
"TEfficiency"]
246 if self.object_a.ClassName() != self.object_b.ClassName():
248 if self.object_a.ClassName()
not in supported_types:
251 if self.object_a.ClassName() ==
"TEfficiency":
253 if self.object_a.GetDimension() > 1:
258 def _raise_has_correct_types(self) -> None:
260 Raise Exception if not the two objects have a) a type supported for
261 comparison and b) can be compared with each other
264 if not self._has_correct_types():
266 "Comparison of {} (Type {}) with {} (Type {}) not "
267 "supported.\nPlease open a GitLab issue (validation "
268 "label) if you need this supported. "
270 raise ObjectsNotSupported(
272 self.object_a.GetName(),
273 self.object_a.ClassName(),
274 self.object_b.GetName(),
275 self.object_b.ClassName(),
279 def _has_compatible_bins(self) -> bool:
281 Check if both ROOT objects have the same amount of bins
282 @return: True if the bins are equal, otherwise False
285 self.object_a.ClassName()
287 == self.object_b.ClassName()
289 nbins_a = self.object_a.GetTotalHistogram().GetNbinsX()
290 nbins_b = self.object_b.GetTotalHistogram().GetNbinsX()
292 nbins_a = self.object_a.GetNbinsX()
293 nbins_b = self.object_b.GetNbinsX()
295 return nbins_a == nbins_b
297 def _raise_has_compatible_bins(self) -> None:
299 Raise Exception if not both ROOT objects have the same amount of bins
302 if not self._has_compatible_bins():
304 "The objects have differing x bin count: {} has {} vs. {} "
307 raise DifferingBinCount(
309 self.object_a.GetName(),
310 self.object_a.GetNbinsX(),
311 self.object_b.GetName(),
312 self.object_b.GetNbinsX(),
317 def _convert_teff_to_hist(teff_a):
319 Convert the content of a TEfficiency plot to a histogram and set
320 the bin content and errors
322 conv_hist = teff_a.GetTotalHistogram()
323 xbin_count = conv_hist.GetNbinsX()
324 xbin_low = conv_hist.GetXaxis().GetXmin()
325 xbin_max = conv_hist.GetXaxis().GetXmax()
328 teff_a.GetName() +
"root_conversion",
336 for i
in range(1, xbin_count):
337 th1.SetBinContent(i, teff_a.GetEfficiency(i))
338 th1.SetBinError(i, teff_a.GetEfficiencyErrorLow(i))
343class PvalueTest(ComparisonBase):
344 """ Test with a pvalue """
348 _default_pvalue_warn = 0.99
352 _default_pvalue_error = 0.01
354 def __init__(self, *args, **kwargs):
355 """ Initialize Pvalue test
358 *args: Positional arguments to ComparisonBase
359 **kwargs: Keyword arguments to ComparisonBase
361 super().__init__(*args, **kwargs)
365 self._pvalue_warn = self.mop.pvalue_warn()
367 self._pvalue_error = self.mop.pvalue_error()
369 if self._pvalue_warn
is None:
370 self._pvalue_warn = self._default_pvalue_warn
371 if self._pvalue_error
is None:
372 self._pvalue_error = self._default_pvalue_error
374 def _get_comparison_result(self) -> str:
375 if self._pvalue
is None:
378 if self._pvalue < self._pvalue_error:
380 elif self._pvalue < self._pvalue_warn:
390 def _get_comparison_result_long(self):
403class Chi2Test(PvalueTest):
406 Perform a Chi2Test for ROOT objects. The chi2 test method is e.g. described
407 in the documentation of TH1::Chi2Test. Basically this class wraps around
408 this Chi2Test function, and takes care that we can call perform these
409 tests for a wider selection of ROOT objects.
412 def __init__(self, *args, **kwargs):
415 :param args: See arguments of :class:`ComparisonBase`
416 :param kwargs: See arguments of :class:`ComparisonBase`
418 super().__init__(*args, **kwargs)
429 def _ensure_zero_error_has_no_content(self, a, b):
431 Ensure there are no bins which have a content set, but 0 error
432 This bin content will be set to 0 to disable this bin completely during
435 nbins = a.GetNbinsX()
436 for ibin
in range(1, nbins + 1):
437 if a.GetBinError(ibin) <= 0.0
and b.GetBinError(ibin) <= 0.0:
440 a.SetBinContent(ibin, 0.0)
441 b.SetBinContent(ibin, 0.0)
444 f
"DEBUG: Warning: Setting bin content of bin {ibin} to zero for both histograms, because both " +
445 "histograms have vanishing errors there.")
447 def _compute(self) -> None:
449 Performs the actual Chi^2 test
452 self._raise_has_correct_types()
453 self._raise_has_compatible_bins()
455 local_object_a = self.object_a
456 local_object_b = self.object_b
459 if self.object_a.ClassName() ==
"TEfficiency":
460 local_object_a = self._convert_teff_to_hist(self.object_a)
461 local_object_b = self._convert_teff_to_hist(self.object_b)
463 print(
"Converting TEfficiency objects to histograms.")
465 nbins = local_object_a.GetNbinsX()
469 f
"{nbins} bin(s) is too few to perform the Chi2 test."
472 weighted_types = [
"TProfile",
"TH1D",
"TH1F"]
473 comp_weight_a = local_object_a.ClassName()
in weighted_types
474 comp_weight_b = local_object_b.ClassName()
in weighted_types
478 first_obj = local_object_a.Clone()
479 second_obj = local_object_b.Clone()
481 if comp_weight_a
and not comp_weight_b:
484 first_obj, second_obj = second_obj, first_obj
487 "Debug: Warning: Switching the two objects, because "
488 "ROOT can only have the first one to be unweighted"
493 if comp_weight_a
and comp_weight_b:
495 elif comp_weight_a
or comp_weight_b:
500 if comp_weight_a
and comp_weight_b:
501 self._ensure_zero_error_has_no_content(first_obj, second_obj)
504 res_chi2 = numpy.array([1], numpy.float64)
505 res_igood = numpy.array([1], numpy.int32)
506 res_ndf = numpy.array([1], numpy.int32)
508 res_pvalue = first_obj.Chi2TestX(
509 second_obj, res_chi2, res_ndf, res_igood, comp_options
513 print(
"Performing our own chi2 test, with bin-by-bin results: ")
515 print_contents_and_errors(first_obj, second_obj)
518 f
"Here's what ROOT's Chi2Test gave us (comp_options: '{comp_options}'): "
521 tp = TablePrinter(3, width=(10, 10, 40))
524 tp.print([
"Key",
"Value",
"Comment"])
529 numpy.asscalar(res_chi2),
530 "Should roughly match above 'Total chi2'",
533 tp.print([
"ndf", numpy.asscalar(res_ndf),
"#Non-empty bins - 1"])
534 tp.print([
"chi2/ndf", numpy.asscalar(res_chi2 / res_ndf),
""])
538 numpy.asscalar(res_igood),
539 "a debug indicator, 0 if all good",
542 tp.print([
"pvalue", res_pvalue,
""])
546 "See https://root.cern.ch/doc/master/classTH1.html for more "
553 "Comparison failed, no Chi^2 could be computed. For "
554 "debugging, you can use the CLI of "
555 "'validation/scripts/validationcomparison.py' on your root "
556 "file and the reference. Run 'validationcomparison.py "
557 "--help' for info. If problem persists, please open "
558 "GitLab issue (validation label)."
560 raise ComparisonFailed(msg)
562 res_chi2ndf = res_chi2 / res_ndf
564 self._pvalue, self._chi2, self._chi2ndf, self._ndf = (
571 def _get_comparison_result_long(self) -> str:
572 if self._pvalue
is None or self._chi2ndf
is None or self._chi2
is None:
574 r"Could not perform $\chi^2$-Test between {{revision1}} "
575 r"and {{revision2}} due to an unknown error. Please "
576 r"submit a bug report."
580 r"Performed $\chi^2$-Test between {{revision1}} "
581 r"and {{revision2}} "
582 r"($\chi^2$ = {chi2:.4f}; NDF = {ndf}; "
583 r"$\chi^2/\text{{{{NDF}}}}$ = {chi2ndf:.4f})."
584 r" <b>p-value: {pvalue:.6f}</b> (p-value warn: {pvalue_warn}, "
585 r"p-value error: {pvalue_error})".format(
588 chi2ndf=self._chi2ndf,
590 pvalue_warn=self._pvalue_warn,
591 pvalue_error=self._pvalue_error,
601class KolmogorovTest(PvalueTest):
602 """ Kolmogorov-Smirnov Test """
604 def __init__(self, *args, **kwargs):
606 Initialize Kolmogorov test.
607 @param args: See arguments of :class:`ComparisonBase`
608 @param kwargs: See arguments of :class:`ComparisonBase`
610 super().__init__(*args, **kwargs)
614 Perform the actual test
617 self._raise_has_correct_types()
618 self._raise_has_compatible_bins()
620 local_object_a = self.object_a
621 local_object_b = self.object_b
624 if self.object_a.ClassName() ==
"TEfficiency":
625 local_object_a = self._convert_teff_to_hist(self.object_a)
626 local_object_b = self._convert_teff_to_hist(self.object_b)
628 print(
"Converting TEfficiency objects to histograms.")
634 self._pvalue = local_object_a.KolmogorovTest(local_object_b, option_str)
636 def _get_comparison_result_long(self) -> str:
637 if self._pvalue
is None:
639 r"Could not perform Kolmogorov test between {{revision1}} "
640 r"and {{revision2}} due to an unknown error. Please submit "
645 r"Performed Kolmogorov test between {{revision1}} "
646 r"and {{revision2}} "
647 r" <b>p-value: {pvalue:.6f}</b> (p-value warn: {pvalue_warn}, "
648 r"p-value error: {pvalue_error})".format(
650 pvalue_warn=self._pvalue_warn,
651 pvalue_error=self._pvalue_error,
661class AndersonDarlingTest(PvalueTest):
662 """ Anderson-Darling test"""
664 def __init__(self, *args, **kwargs):
666 Initialize Anderson-Darling test.
667 @param args: See arguments of :class:`ComparisonBase`
668 @param kwargs: See arguments of :class:`ComparisonBase`
670 super().__init__(*args, **kwargs)
674 Perform the actual test
677 self._raise_has_correct_types()
682 local_object_a = self.object_a
683 local_object_b = self.object_b
686 if self.object_a.ClassName() ==
"TEfficiency":
687 local_object_a = self._convert_teff_to_hist(self.object_a)
688 local_object_b = self._convert_teff_to_hist(self.object_b)
690 print(
"Converting TEfficiency objects to histograms.")
696 self._pvalue = local_object_a.AndersonDarlingTest(
697 local_object_b, option_str
700 def _get_comparison_result_long(self) -> str:
701 if self._pvalue
is None:
703 r"Could not perform-Anderson Darling test between "
704 r"{{revision1}} and {{revision2}} due to an unknown error."
705 r" Please support a bug report."
709 r"Performed Anderson-Darling test between {{revision1}} "
710 r"and {{revision2}} "
711 r" <b>p-value: {pvalue:.6f}</b> (p-value warn: {pvalue_warn}, "
712 r"p-value error: {pvalue_error})".format(
714 pvalue_warn=self._pvalue_warn,
715 pvalue_error=self._pvalue_error,
726 """ A tiny class to print columns of fixed width numbers. """
728 def __init__(self, ncols, width=None):
731 @param ncols: Number of columns
732 @param width: Width of each column. Either int or list.
738 if isinstance(width, int):
740 self.widths = [width] * ncols
741 elif isinstance(width, list)
or isinstance(width, tuple):
747 """ Total width of the table """
750 width += sum(self.widths)
752 width += (self.ncols - 1) * 3
757 def print_divider(self, char="="):
758 """ Print a divider made up from repeated chars """
759 print(char * self.tot_width)
761 def print(self, cols):
762 """ Print one row """
763 assert len(cols) == self.ncols
765 for icol, col
in enumerate(cols):
766 width = self.widths[icol]
767 if isinstance(col, int):
768 form = f
"{{:{width}d}}"
769 out.append(form.format(col))
770 elif isinstance(col, float):
771 form = f
"{{:{width}.{width // 2}f}}"
772 out.append(form.format(col))
776 col = col[:width].rjust(width)
778 print(
"| " +
" | ".join(out) +
" |")
781def print_contents_and_errors(obj_a, obj_b):
783 Print contents, errors and chi2 deviation for each bin as well as
784 some other information about two TH1-like objects.
785 @param obj_a: First TH1-like object
786 @param obj_b: Second TH1-like object
789 nbins = obj_a.GetNbinsX()
791 total_a = sum([obj_a.GetBinContent(ibin)
for ibin
in range(0, nbins + 2)])
792 total_b = sum([obj_b.GetBinContent(ibin)
for ibin
in range(0, nbins + 2)])
794 print(f
"Total events/summed weights in object 1: {total_a:10.5f}")
795 print(f
"Total events/summed weights in object 2: {total_b:10.5f}")
802 cp.print([
"ibin",
"a",
"err a",
"b",
"err b",
"chi2"])
804 for ibin
in range(1, nbins + 1):
805 content_a = obj_a.GetBinContent(ibin)
806 content_b = obj_b.GetBinContent(ibin)
807 error_a = obj_a.GetBinError(ibin)
808 error_b = obj_b.GetBinError(ibin)
812 chi2 = (total_b * content_a - total_a * content_b) ** 2 / (
813 total_b ** 2 * error_a ** 2 + total_a ** 2 * error_b ** 2
816 except ZeroDivisionError:
818 cp.print([ibin, content_a, error_a, content_b, error_b, chi2])
822 print(f
"Total chi2: {chi2_tot:10.5f}")
831 """ A small command line interface for debugging purposes. """
837 "For testing purposes: Run the chi2 comparison with objects from "
840 parser = argparse.ArgumentParser(desc)
842 _ =
"Rootfile to read the first object from"
843 parser.add_argument(
"rootfile_a", help=_)
845 _ =
"Name of object inside first rootfile."
846 parser.add_argument(
"name_a", help=_)
848 _ =
"Rootfile to read the second object from"
849 parser.add_argument(
"rootfile_b", help=_)
851 _ =
"Name of object inside second rootfile."
852 parser.add_argument(
"name_b", help=_)
854 args = parser.parse_args()
859 if not os.path.exists(args.rootfile_a):
860 raise ValueError(f
"Could not find '{args.rootfile_a}'.")
862 if not os.path.exists(args.rootfile_b):
863 raise ValueError(f
"Could not find '{args.rootfile_b}'.")
865 rootfile_a = ROOT.TFile(args.rootfile_a)
866 obj_a = rootfile_a.Get(args.name_a)
869 f
"Could not find object '{args.name_a}' "
870 f
"in file '{args.rootfile_a}'."
873 rootfile_b = ROOT.TFile(args.rootfile_b)
874 obj_b = rootfile_b.Get(args.name_b)
877 f
"Could not find object '{args.name_b}' "
878 f
"in file '{args.rootfile_b}'."
884 test = Chi2Test(obj_a, obj_b, debug=
True)
885 test.ensure_compute()
887 print(
"If you see this message, then no exception was thrown.")
896if __name__ ==
"__main__":