Belle II Software development
classversion.py
1
8"""
9b2test_utils.classversion - Helper functions to inspect and verify ROOT dictionaries
10------------------------------------------------------------------------------------
11
12This module contains some functions to make sure classes which have a ROOT
13dictionary are healthy by allowing to check that
14
15* All base classes are fully defined and not forward declared
16* All members have a dictionary
17* The class version and checksum is as expected to avoid mistakes where the
18 classdef is not increased after class changes
19
20Users should only need to call ``b2code-classversion-check`` and
21``b2code-classversion-update``. The documentation of
22``b2code-classversion-check`` also contains a bit more detailed explanation of
23class versions and checksums
24"""
25
26import re
27import os
28import sys
29from basf2 import B2INFO, B2ERROR, B2WARNING
30
31
32class ClassVersionError(Exception):
33 """Exception to report class version errors"""
34
35
36class ErrorWithExtraVariables(Exception):
37 """Exception class with extra keyword arguments to show in log message"""
38
39
40 def __init__(self, *args, **argk):
41 super().__init__(*args)
42
43 self.variables = argk
44
45
46def check_base_classes(tclass):
47 """Recursively check all base classes of a TClass to make sure all are well defined"""
48 bases = tclass.GetListOfBases()
49 if bases is None:
50 raise ClassVersionError("Cannot get list of base classes.")
51 for base in bases:
52 baseclass = base.GetClassPointer()
53 if not baseclass:
54 raise ClassVersionError(f"incomplete base class {base.GetName()}")
55 check_base_classes(baseclass)
56
57
58def check_dictionary(classname):
59 """Make sure we have a dictionary for the class and all its members"""
60 # Always avoid the top-level 'import ROOT'.
61 import ROOT # noqa
62 tclass = ROOT.TClass.GetClass(classname)
63 if not tclass:
64 raise ClassVersionError("Cannot find TClass object")
65 streamerinfo = tclass.GetStreamerInfo()
66 if streamerinfo:
67 for element in streamerinfo.GetElements():
68 elementclass = element.GetClassPointer()
69 if elementclass and not elementclass.IsLoaded():
70 raise ClassVersionError(f"Missing dictionary for member {element.GetName()}, "
71 f"type {elementclass.GetName()}")
72
73
74def get_class_version(classname):
75 """Get the Class version and checksum for a fully qualified C++ class name"""
76 # Always avoid the top-level 'import ROOT'.
77 import ROOT # noqa
78 tclass = ROOT.TClass.GetClass(classname)
79 if not tclass:
80 raise ClassVersionError("Cannot find TClass object")
81 # good time to also check base classes
82 check_base_classes(tclass)
83 version = tclass.GetClassVersion()
84 checksum = tclass.GetCheckSum()
85 return version, checksum
86
87
88def iterate_linkdef(filename):
89 """Iterate through the lines of a ROOT linkdef file.
90
91 This function is a generator that foreach line returns a tuple where the
92 first element is True if there was a class link pragma on that line and
93 False otherwise.
94
95 If it was a class link pragma the second element will be a tuple
96 ``(line, startcolumn, classname, linkoptions, options)`` where
97
98 * ``line`` is the content of the line
99 * ``startcolumn`` is the column the content of the comment started in the line.
100 Everything before should be kept as is
101 * ``classname`` is the class name
102 * ``clingflags`` is the link flags requested by either the optional +-! after
103 the class or the ``options=`` in front of the class and a set of the value
104 "evolution", "nostreamer", and "noinputoper"
105 * ``options`` is a dictionary of the options found in the comment. The
106 content of the comment is split at commas and the result are the options.
107 If a options has a value after ``=`` that will be assigned, otherwise the
108 value is None. For example a comment ``// implicit, version=bar`` would
109 return ``{'implicit':None, 'version':'bar}``
110
111 If it wasn't link pragma the second element is just the unmodified line
112 """
113 # regular expression to find link class pragmas
114 class_regex = re.compile(r"^\s*#pragma\s+link\s+C\+\+\s+(?:options=(.*?)\s)?\s*class\s+(.*?)([+\-!]*);(\s*//\s*(.*))?$")
115 with open(filename) as linkdef:
116 for line in linkdef:
117 match = class_regex.match(line)
118 # if the line doesn't match return it unchanged
119 if not match:
120 yield False, line
121 else:
122 # Otherwise extract the fields
123 linkflags, classname, flags, comment, content = match.groups()
124 start = match.start(4) if comment else len(line) - 1 # no comment: start of comment is end of line minus newline
125 # parse the linkflags
126 clingflags = set()
127 for char in flags:
128 clingflags.add({"+": "evolution", "-": "nostreamer", "!": "noinputoper"}.get(char))
129 if linkflags is not None:
130 for flag in linkflags.split(","):
131 clingflags.add(flag.strip())
132 # and parse the comment
133 options = {}
134 if content is not None:
135 optionlist = (e.strip().split("=", 1) for e in content.split(","))
136 options = {e[0]: e[1] if len(e) > 1 else None for e in optionlist}
137 # and return the separate parts
138 yield True, (line, start, classname, clingflags, options)
139
140
141def check_linkdef(filename, message_style="belle2"):
142 """Check a linkdef file for expected versions/checksums
143
144 * If the class has ``version=N, checksum=0x1234`` in the comment in the
145 linkdef compare that to what ROOT has created
146 * If not warn about the missing version and checksum
147 * But skip classes which are not stored to file.
148 """
149 errors = 0
150 relative_filename = os.path.relpath(filename)
151
152 def print_message(severity, text, **argk):
153 """Print a message similar to GCC or Clang: ``filename:line:column: error: text``
154 so that it gets picked up by the build system
155 """
156 nonlocal message_style, errors, relative_filename, nr, column, classname
157
158 if message_style == "gcc":
159 print(f"{relative_filename}:{nr}:{column}: {severity}: class {classname}: {text}", file=sys.stderr, flush=True)
160 else:
161 variables = {"linkdef": f"{relative_filename}:{nr}", "classname": classname}
162 variables.update(argk)
163 if severity == "error":
164 B2ERROR(text, **variables)
165 else:
166 B2WARNING(text, **variables)
167
168 if severity == "error":
169 errors += 1
170
171 def get_int(container, key):
172 """Get the value from a container if it exists and try to convert to int.
173 Otherwise show error"""
174 value = options.get(key, None)
175 if value is not None:
176 try:
177 return int(value, 0)
178 except ValueError as e:
179 print_message("error", f"expected {key} is not valid: {e}")
180
181 # loop over all lines in the linkdef
182 for nr, (isclass, content) in enumerate(iterate_linkdef(filename)):
183 # and ignore everything not a class
184 if not isclass:
185 continue
186
187 line, column, classname, clingflags, options = content
188
189 # no need to check anything else if we don't have storage enabled
190 if "nostreamer" in clingflags:
191 if "evolution" in clingflags:
192 print_message("error", "both, no streamer and class evolution requested.")
193 # current ROOT code lets "evolution win so lets continue as if
194 # nostreamer wouldn't be present, we flagged the error"
195 else:
196 continue
197 # but warn if we use old style streamer without evolution rules
198 elif "evolution" not in clingflags:
199 print_message("warning", "using old ROOT3 streamer format without evolution. "
200 "Please add + or - (for classes not to be written to file) after the classname.")
201
202 # check if we can actually load the class
203 try:
204 version, checksum = get_class_version(classname)
205 except ClassVersionError as e:
206 print_message("error", e)
207
208 # This class seems to be intended to be serialized so make sure we can
209 if "version" in locals() and version != 0:
210 try:
211 check_dictionary(classname)
212 except ClassVersionError as e:
213 print_message("error", e)
214 else:
215 continue
216
217 # and check expected version/checksum
218 expected_version = get_int(options, "version")
219 expected_checksum = get_int(options, "checksum")
220 if expected_version is None or expected_checksum is None:
221 print_message("warning", "no expected version and checksum, cannot spot changes to class")
222 continue
223
224 # And now we know what we have ... so check the version
225 if version != expected_version:
226 print_message("error", "class version has changed, please update the linkdef",
227 expected_version=expected_version, actual_Version=version)
228 # And if it's non-zero also check the checksum. Zero means deactivated streamer
229 elif checksum != expected_checksum and version != 0:
230 print_message("error", "class checksum has changed! "
231 "Did you forget increasing the linkdef version?",
232 expected_checksum=expected_checksum, actual_checksum=checksum)
233
234 return errors == 0
235
236
237def verify_classlink_version(classname, options):
238 """This function verifies the version and checksum of a class from a given
239 linkdef classlink line and returns new values if they can be updated
240
241 Parameters:
242 classname (str): The classname as in the linkdef file
243 options (dict): The options currently present in the linkdef file.
244 Previous version and checksum are taken from here if they exist
245
246 * If there's nothing to do it will return None
247 * If there's any problem it will raise an exception
248 * Otherwise it will return a dictionary containing the new ``version`` and
249 ``checksum`` to be put in the file
250 """
251 # Try to get the actual values, if that fails it will raise which is handled
252 # by caller
253 version, checksum = get_class_version(classname)
254
255 # We have the actual values but what do we know about previous
256 # values?
257 if "version" in options:
258 previous_version = int(options['version'], 0)
259 # Don't allow decrease of class version
260 if previous_version > version:
261 raise ErrorWithExtraVariables(
262 "actual class version lower than previous version in linkdef. "
263 "Class version cannot decrease",
264 previous_version=previous_version, actual_version=version,
265 )
266 # Also please don't just increase by a massive amount unless the previous
267 # version number was negative for "no explicit number"
268 elif previous_version > -1 and version > (previous_version + 1):
269 raise ErrorWithExtraVariables(
270 "actual class version increased by more than one compared to the "
271 "previous version set in linkdef. Please increase ClassDef in single steps",
272 previous_version=previous_version, actual_version=version,
273 )
274 # Otherwise if we have a checksum make sure it is the same
275 # if the version didn't change
276 if "checksum" in options:
277 previous_checksum = int(options['checksum'], 0)
278 if version == previous_version and checksum != previous_checksum:
279 raise ErrorWithExtraVariables(
280 "checksum changed but class version didn't change. "
281 "Please increase the class version if you modified the class contents",
282 version=version, actual_checksum=hex(checksum),
283 previous_checksum=hex(previous_checksum),
284 )
285 # but if both are the same just keep the line unchanged
286 elif (version, checksum) == (previous_version, previous_checksum):
287 return
288
289 # Ok, we have our values, return them
290 return {"version": version, "checksum": hex(checksum)}
291
292
293def update_linkdef(filename):
294 """
295 Update a given linkdef file and update all the versions and checksums
296
297 Parameters:
298 filename (str): The linkdef.h file to be updated
299 """
300
301 lines = []
302 # Loop over all lines
303 for nr, (isclass, content) in enumerate(iterate_linkdef(filename)):
304 # And if it's not a class link pragma keep the line unmodified
305 if not isclass:
306 lines.append(content)
307 continue
308
309 # Otherwise check if we need a version and checksum
310 line, column, classname, clingflags, options = content
311 try:
312 # only for classes to be streamed ...
313 if "nostreamer" not in clingflags:
314 # verify the version ...
315 version = verify_classlink_version(classname, options)
316 if version is not None:
317 options.update(version)
318 B2INFO(f"Updating ROOT class version and checksum for {classname}",
319 linkdef=f"{os.path.relpath(filename)}:{nr}", **version)
320 # convert the options to a comment string
321 optionstr = []
322 for key, value in sorted(options.items()):
323 optionstr.append(key + (f"={value}" if value is not None else ""))
324 comment = " // " + ", ".join(optionstr)
325 lines.append(f"{line[:column]}{comment}\n")
326 continue
327
328 except Exception as e:
329 variables = {"classname": classname, "linkdef": f"{os.path.relpath(filename)}:{nr}"}
330 if isinstance(e, ErrorWithExtraVariables):
331 variables.update(e.variables)
332 B2ERROR(str(e), **variables)
333
334 # Finally, keep the line in case of errors or no change
335 lines.append(line)
336
337 # And then replace the file
338 with open(filename, "w") as newlinkdef:
339 newlinkdef.writelines(lines)
__init__(self, *args, **argk)
Initialize the class.