12This tool provides a command line interface to all the tasks related to the
13:ref:`Conditions database <conditionsdb_overview>`: manage globaltags and iovs
14as well as upload new payloads or download of existing payloads.
16The usage of this tool is similar to git: there are sub commands like for
17example ``tag`` which groups all actions related to the management of
18globaltags. All the available commands are listed below.
20While the read access to the conditions database is always allowed (e.g.
21for downloading globaltags and payloads), users need a valid JSON Web Token
22(JWT) to authenticate to the conditions database when using this tool for
23creating/manpipulating globaltags or uploading payloads. For practical purposes,
24it is only necessary to know that a JWT is a string containing encrypted
25information, and that string is stored in a file. More information about what
26a JWT is can be found on
27`Wikipedia <https://en.wikipedia.org/wiki/JSON_Web_Token>`_.
31 By default, users can only create globaltags whose name starts with
32 ``user_<username>_`` or ``temp_<username>_``, where ``username`` is the
35The tool automatically queries the JWT issuing server
36(https://token.belle2.org) and gets a valid token by asking the B2MMS username
37and password. The issued "default" JWT has a validity of 1 hour; after it
38expires, a new JWT needs to be obtained for authenticating the conditions
39database. When retrieved by this tool, the JWT is stored locally in the file
40``${HOME}/b2cdb_${BELLE2_USER}.token``.
42Some selected users (e.g. the calibration managers) are granted a JWT with an
43extended validity (30 days) to allow smooth operations with some automated
44workflows. Such "extended" JWTs are issued by a different server
45(https://token.belle2.org/extended). B2MMS username and password are necessary
46for getting the extended JWT. The extended JWT differs from the default JWT
47only by its validity and can be obtained only by manually querying the
48alternative server. If queried via web browser, a file containing the extended
49JWT will be downloaded in case the user is granted it. The server can also be
50queried via command line using
51``wget --user USERNAME --ask-password --no-check-certificate https://token.belle2.org/extended``
52or ``curl -u USERNAME -k https://token.belle2.org/extended``.
54If the environment variable ``${BELLE2_CDB_AUTH_TOKEN}`` is defined and points
55to a file containing a valid JWT, the ``b2conditionsdb`` tools use this token
56to authenticate with the CDB instead of querying the issuing server. This is
57useful when using an extended token. Please note that, in case the JWT to which
58``${BELLE2_CDB_AUTH_TOKEN}`` points is not valid anymore, the
59``b2conditionsdb`` tools will attempt to get a new one and store it into
60``${BELLE2_CDB_AUTH_TOKEN}``. If a new extended token is needed, it has to be
61manually obtained via https://token.belle2.org/extended and stored into
62``${BELLE2_CDB_AUTH_TOKEN}``.
86from basf2
import B2ERROR, B2WARNING, B2INFO, LogLevel, LogInfo, logging, \
89from terminal_utils
import Pager
90from dateutil.parser
import parse
as parse_date
91from getpass
import getuser
92from conditions_db
import ConditionsDB, enable_debugging, encode_name, PayloadInformation, set_cdb_authentication_token
99from conditions_db.cli_management import command_tag_merge, command_tag_runningupdate, command_iovs, command_iovs_delete, command_iovs_copy, command_iovs_modify
102def escape_ctrl_chars(name):
103 """Remove ANSI control characters from strings"""
105 if not hasattr(escape_ctrl_chars,
"_regex"):
106 escape_ctrl_chars._regex = re.compile(
"[\x00-\x1f\x7f-\x9f]")
110 if match.group(0).isspace():
111 return match.group(0)
112 return f
"\\x{ord(match.group(0)):02x}"
114 return escape_ctrl_chars._regex.sub(escape, name)
117def command_tag(args, db=None):
119 List, show, create, modify or clone globaltags.
121 This command allows to list, show, create modify or clone globaltags in the
122 central database. If no other command is given it will list all tags as if
123 ``%(prog)s show`` was given.
129 command_tag_list(args, db)
132def command_tag_list(args, db=None):
134 List all available globaltags.
136 This command allows to list all globaltags, optionally limiting the output
137 to ones matching a given search term. By default invalidated globaltags
138 will not be included in the list, to show them as well please add
139 ``--with-invalid`` as option. Alternatively one can use ``--only-published``
140 to show only tags which have been published
142 If the ``--regex`` option is supplied the search term will be interpreted
143 as a Python regular expression where the case is ignored.
146 tagfilter = ItemFilter(args)
149 args.add_argument(
"--detail", action=
"store_true", default=
False,
150 help=
"show detailed information for all tags instead "
152 group = args.add_mutually_exclusive_group()
153 group.add_argument(
"--with-invalid", action=
"store_true", default=
False,
154 help=
"if given also invalidated tags will be shown")
155 group.add_argument(
"--only-published", action=
"store_true", default=
False,
156 help=
"if given only published tags will be shown")
157 tagfilter.add_arguments(
"tags")
161 if not tagfilter.check_arguments():
164 req = db.request(
"GET",
"/globalTags", f
"Getting list of globaltags{tagfilter}")
168 for item
in req.json():
169 if not tagfilter.check(item[
"name"]):
171 tagstatus = item[
"globalTagStatus"][
"name"]
172 if not getattr(args,
"with_invalid",
False)
and tagstatus ==
"INVALID":
174 if getattr(args,
"only_published",
False)
and tagstatus !=
"PUBLISHED":
179 taglist.sort(key=
lambda x: x[
"name"])
183 with Pager(f
"List of globaltags{tagfilter}{' (detailed)' if getattr(args, 'detail', False) else ''}",
True):
185 if getattr(args,
"detail",
False):
186 print_globaltag(db, item)
191 escape_ctrl_chars(item.get(
"description",
"")),
192 item[
"globalTagType"][
"name"],
193 item[
"globalTagStatus"][
"name"],
194 item[
"payloadCount"],
197 if not getattr(args,
"detail",
False):
198 table.insert(0, [
"id",
"name",
"description",
"type",
"status",
"# payloads"])
199 pretty_print_table(table, [-10, 0,
"*", -10, -10, -10], min_flexible_width=32)
202def print_globaltag(db, *tags):
203 """ Print detailed globaltag information for the given globaltags side by side"""
204 results = [[
"id"], [
"name"], [
"description"], [
"type"], [
"status"],
205 [
"# payloads"], [
"created"], [
"modified"], [
"modified by"]]
210 if isinstance(info, str):
212 req = db.request(
"GET", f
"/globalTag/{encode_name(info)}",
213 f
"Getting info for globaltag {info}")
214 except ConditionsDB.RequestError
as e:
222 created = parse_date(info[
"dtmIns"])
223 modified = parse_date(info[
"dtmMod"])
224 results[0].append(str(info[
"globalTagId"])),
225 results[1].append(info[
"name"]),
226 results[2].append(escape_ctrl_chars(info.get(
"description",
"")))
227 results[3].append(info[
"globalTagType"][
"name"])
228 results[4].append(info[
"globalTagStatus"][
"name"]),
229 results[5].append(info[
"payloadCount"]),
230 results[6].append(created.astimezone(tz=
None).strftime(
"%Y-%m-%d %H:%M:%S local time"))
231 results[7].append(modified.astimezone(tz=
None).strftime(
"%Y-%m-%d %H:%M:%S local time"))
232 results[8].append(escape_ctrl_chars(info[
"modifiedBy"]))
234 ntags = len(results[0]) - 1
236 pretty_print_table(results, [11] + [
'*'] * ntags,
True)
240def change_state(db, tag, state, force=False):
241 """Change the state of a global tag
243 If the new state is not revertible then ask for confirmation
245 state = state.upper()
246 if state
in [
"INVALID",
"PUBLISHED"]
and not force:
247 name = input(f
"ATTENTION: Marking a tag as {state} cannot be undone.\n"
248 "If you are sure you want to continue it please enter the tag name again: ")
250 B2ERROR(
"Names don't match, aborting")
253 db.request(
"PUT", f
"/globalTag/{encode_name(tag)}/updateGlobalTagStatus/{state}",
254 f
"Changing globaltag state {tag} to {state}")
257def command_tag_show(args, db=None):
259 Show details about globaltags
261 This command will show details for the given globaltags like name,
262 description and number of payloads.
270 args.add_argument(
"tag", metavar=
"TAGNAME", nargs=
"+", help=
"globaltags to show")
275 with Pager(
"globaltag Information",
True):
277 ntags += print_globaltag(db, tag)
280 return len(args.tag) - ntags
283def command_tag_create(args, db=None):
285 Create a new globaltag
287 This command creates a new globaltag in the database with the given name
288 and description. The name can only contain alphanumeric characters and the
293 By default, users can only create globaltags whose name starts with
294 ``user_<username>_`` or ``temp_<username>_``, where ``username`` is the
299 args.add_argument(
"type", metavar=
"TYPE", help=
"type of the globaltag to create, usually one of DEV or RELEASE")
300 args.add_argument(
"tag", metavar=
"TAGNAME", help=
"name of the globaltag to create")
301 args.add_argument(
"description", metavar=
"DESCRIPTION", help=
"description of the globaltag")
302 args.add_argument(
"-u",
"--user", metavar=
"USER", help=
"username who created the tag. "
303 "If not given we will try to supply a useful default")
306 set_cdb_authentication_token(db, args.auth_token)
309 info = {
"name": args.tag,
"description": args.description,
"modifiedBy": args.user,
"isDefault":
False}
311 if args.user
is None:
312 info[
"modifiedBy"] = os.environ.get(
"BELLE2_USER", getuser())
314 typeinfo = db.get_globalTagType(args.type)
318 req = db.request(
"POST", f
"/globalTag/{encode_name(typeinfo['name'])}",
319 "Creating globaltag {name}".format(**info),
321 B2INFO(
"Successfully created globaltag {name} (id={globalTagId})".format(**req.json()))
324def command_tag_modify(args, db=None):
326 Modify a globaltag by changing name or description
328 This command allows to change the name or description of an existing globaltag.
329 You can supply any combination of -n,-d,-t and only the given values will be changed
332 args.add_argument(
"tag", metavar=
"TAGNAME", help=
"globaltag to modify")
333 args.add_argument(
"-n",
"--name", help=
"new name")
334 args.add_argument(
"-d",
"--description", help=
"new description")
335 args.add_argument(
"-t",
"--type", help=
"new type of the globaltag")
336 args.add_argument(
"-u",
"--user", metavar=
"USER", help=
"username who created the tag. "
337 "If not given we will try to supply a useful default")
338 args.add_argument(
"-s",
"--state", help=
"new globaltag state, see the command ``tag state`` for details")
341 set_cdb_authentication_token(db, args.auth_token)
344 req = db.request(
"GET", f
"/globalTag/{encode_name(args.tag)}",
345 f
"Getting info for globaltag {args.tag}")
349 old_name = info[
"name"]
351 for key
in [
"name",
"description"]:
352 value = getattr(args, key)
353 if value
is not None and value != info[key]:
357 info[
"modifiedBy"] = os.environ.get(
"BELLE2_USER", os.getlogin())
if args.user
is None else args.user
359 if args.type
is not None:
361 typeinfo = db.get_globalTagType(args.type)
365 if info[
'globalTagType'] != typeinfo:
366 info[
"globalTagType"] = typeinfo
371 db.request(
"PUT",
"/globalTag",
372 "Modifying globaltag {} (id={globalTagId})".format(old_name, **info),
375 if args.state
is not None:
376 name = args.name
if args.name
is not None else old_name
377 return change_state(db, name, args.state)
380def command_tag_clone(args, db=None):
382 Clone a given globaltag including all IoVs
384 This command allows to clone a given globaltag with a new name but still
385 containing all the IoVs defined in the original globaltag.
389 args.add_argument(
"tag", metavar=
"TAGNAME", help=
"globaltag to be cloned")
390 args.add_argument(
"name", metavar=
"NEWNAME", help=
"name of the cloned globaltag")
393 set_cdb_authentication_token(db, args.auth_token)
396 req = db.request(
"GET", f
"/globalTag/{encode_name(args.tag)}",
397 f
"Getting info for globaltag {args.tag}")
401 req = db.request(
"POST",
"/globalTags/{globalTagId}".format(**info),
402 "Cloning globaltag {name} (id={globalTagId})".format(**info))
406 cloned_info = req.json()
407 cloned_info[
"name"] = args.name
409 db.request(
"PUT",
"/globalTag",
"Renaming globaltag {name} (id={globalTagId})".format(**cloned_info),
413def command_tag_state(args, db):
415 Change the state of a globaltag.
417 This command changes the state of a globaltag to the given value.
419 Usually the valid states are
422 Tag can be modified, payloads and iovs can be created and deleted. This
423 is the default state for new or cloned globaltags and is not suitable
424 for use in data analysis
426 Can be transitioned to TESTING, RUNNING
429 Tag cannot be modified and is suitable for testing but can be reopened
431 Can be transitioned to VALIDATED, OPEN
434 Tag cannot be modified and has been tested.
436 Can be transitioned to PUBLISHED, OPEN
439 Tag cannot be modified and is suitable for user analysis
441 Can only be transitioned to INVALID
444 Tag can only be modified by adding new runs, not modifying the payloads
448 Tag is invalid and should not be used for anything.
450 This state is end of life for a globaltag and cannot be transitioned to
453 .. note:: Version added: release-04-00-00
456 args.add_argument(
"tag", metavar=
"TAGNAME", help=
"globaltag to be changed")
457 args.add_argument(
"state", metavar=
"STATE", help=
"new state for the globaltag")
458 args.add_argument(
"--force", default=
False, action=
"store_true", help=argparse.SUPPRESS)
461 set_cdb_authentication_token(db, args.auth_token)
463 return change_state(db, args.tag, args.state, args.force)
466def remove_repeated_values(table, columns, keep=None):
467 """Strip repeated values from a table of values
469 This function takes a table (a list of lists with all the same length) and
470 removes values in certain columns if they are identical in consecutive rows.
472 It does this in a dependent way only if the previous columns are identical
473 will it continue stripping further columns. For example, given the table ::
482 If we want to remove duplicates in all columns in order it would look like this:
484 >>> remove_repeated_values(table, [0,1])
492 But we can give selected columns to strip one after the other
494 >>> remove_repeated_values(table, [1,0])
502 In addition, we might want to only strip some columns if previous columns
503 were identical but keep the values of the previous column. For this one can
506 >>> remove_repeated_values(table, [0,1,2], keep=[0])
515 table (list(list(str))): 2d table of values
516 columns (list(int)): indices of columns to consider in order
517 keep (set(int)): indices of columns to not strip
519 last_values = [
None] * len(columns)
520 for row
in table[1:]:
521 current_values = [row[i]
for i
in columns]
522 for i, curr, last
in zip(columns, current_values, last_values):
526 if keep
and i
in keep:
531 last_values = current_values
534def command_diff(args, db):
536 Compare two globaltags
538 This command allows to compare two globaltags. It will show the changes in
539 a format similar to a unified diff but by default it will not show any
540 context, only the new or removed payloads. Added payloads are marked with a
541 ``+`` in the first column, removed payloads with a ``-``
543 If ``--full`` is given it will show all payloads, even the ones common to
544 both globaltags. The differences can be limited to a given run and
545 limited to a set of payloads names using ``--filter`` or ``--exclude``. If
546 the ``--regex`` option is supplied the search term will be interpreted as a
547 python regular expression where the case is ignored.
549 .. note:: Version changed: release-03-00-00
551 modified output structure and added ``--human-readable``
552 .. note:: Version changed: after release-04-00-00
554 added parameter ``--checksums`` and ``--show-ids``
556 iovfilter = ItemFilter(args)
558 args.add_argument(
"--full", default=
False, action=
"store_true",
559 help=
"If given print all iovs, also those which are the same in both tags")
560 args.add_argument(
"--run", type=int, nargs=2, metavar=
"N", help=
"exp and run numbers "
561 "to limit showing iovs to a ones present in a given run")
562 args.add_argument(
"--human-readable", default=
False, action=
"store_true",
563 help=
"If given the iovs will be written in a more human friendly format. "
564 "Also repeated payload names will be omitted to create a more readable listing.")
565 args.add_argument(
"--checksums", default=
False, action=
"store_true",
566 help=
"If given don't show the revision number but the md5 checksum")
567 args.add_argument(
"--show-ids", default=
False, action=
"store_true",
568 help=
"If given also show the payload and iov ids for each iov")
570 args.add_argument(
"tagA", metavar=
"TAGNAME1", help=
"base for comparison")
571 args.add_argument(
"tagB", metavar=
"TAGNAME2", help=
"tagname to compare")
572 args.add_argument(
"--run-range", nargs=4, default=
None, type=int,
573 metavar=(
"FIRST_EXP",
"FIRST_RUN",
"FINAL_EXP",
"FINAL_RUN"),
574 help=
"Can be four numbers to limit the run range to be compared"
575 "Only iovs overlapping, even partially, with this range will be considered.")
576 iovfilter.add_arguments(
"payloads")
580 if not iovfilter.check_arguments():
583 with Pager(f
"Differences between globaltags {args.tagA} and {args.tagB}{iovfilter}",
True):
584 print(
"globaltags to be compared:")
585 ntags = print_globaltag(db, args.tagA, args.tagB)
590 e
for e
in db.get_all_iovs(
592 message=str(iovfilter),
593 run_range=args.run_range)
if iovfilter.check(
596 e
for e
in db.get_all_iovs(
598 message=str(iovfilter),
599 run_range=args.run_range)
if iovfilter.check(
602 B2INFO(
"Comparing contents ...")
603 diff = difflib.SequenceMatcher(a=listA, b=listB)
604 table = [[
"",
"Name",
"Rev" if not args.checksums
else "Checksum"]]
605 columns = [1,
"+", -8
if not args.checksums
else -32]
607 if args.human_readable:
611 table[0] += [
"First Exp",
"First Run",
"Final Exp",
"Final Run"]
612 columns += [6, 6, 6, 6]
615 table[0] += [
"IovId",
"PayloadId"]
618 def add_payloads(opcode, payloads):
619 """Add a list of payloads to the table, filling the first column with opcode"""
621 row = [opcode, p.name, p.revision
if not args.checksums
else p.checksum]
622 if args.human_readable:
623 row += [p.readable_iov()]
628 row += [p.iov_id, p.payload_id]
631 for tag, i1, i2, j1, j2
in diff.get_opcodes():
635 add_payloads(
" ", listB[j1:j2])
636 if tag
in [
"delete",
"replace"]:
637 add_payloads(
"-", listA[i1:i2])
638 if tag
in [
"insert",
"replace"]:
639 add_payloads(
"+", listB[j1:j2])
641 if args.human_readable:
645 remove_repeated_values(table, [0, 1, 2] + ([-1]
if args.show_ids
else []), keep=[0])
647 def color_row(row, widths, line):
648 if not LogPythonInterface.terminal_supports_colors():
650 begin = {
'+':
'\x1b[32m',
'-':
'\x1b[31m'}.get(row[0],
"")
652 return begin + line + end
657 print(f
" Differences between {args.tagA} and {args.tagB}")
658 pretty_print_table(table, columns, transform=color_row,
659 hline_formatter=
lambda w:
" " + (w - 1) *
'-')
662def command_iov(args, db):
664 List all IoVs defined in a globaltag, optionally limited to a run range
666 This command lists all IoVs defined in a given globaltag. The list can be
667 limited to a given run and optionally searched using ``--filter`` or
668 ``--exclude``. If the ``--regex`` option is supplied the search term will
669 be interpreted as a Python regular expression where the case is ignored.
671 .. note:: Version changed: release-03-00-00
673 modified output structure and added ``--human-readable``
674 .. note:: Version changed: after release-04-00-00
676 added parameter ``--checksums`` and ``--show-ids``
677 .. note:: Version changed: after release-08-00-04
679 added parameter ``--run-range``
682 iovfilter = ItemFilter(args)
685 args.add_argument(
"tag", metavar=
"TAGNAME", help=
"globaltag for which the the IoVs should be listed")
686 args.add_argument(
"--run", type=int, nargs=2, metavar=
"N", help=
"exp and run numbers "
687 "to limit showing iovs to a ones present in a given run")
688 args.add_argument(
"--detail", action=
"store_true", default=
False,
689 help=
"if given show a detailed information for all "
690 "IoVs including details of the payloads")
691 args.add_argument(
"--human-readable", default=
False, action=
"store_true",
692 help=
"If given the iovs will be written in a more human friendly format. "
693 "Also repeated payload names will be omitted to create a more readable listing.")
694 args.add_argument(
"--checksums", default=
False, action=
"store_true",
695 help=
"If given don't show the revision number but the md5 checksum")
696 args.add_argument(
"--show-ids", default=
False, action=
"store_true",
697 help=
"If given also show the payload and iov ids for each iov")
698 args.add_argument(
"--run-range", nargs=4, default=
None, type=int,
699 metavar=(
"FIRST_EXP",
"FIRST_RUN",
"FINAL_EXP",
"FINAL_RUN"),
700 help=
"Can be four numbers to limit the run range to be shown"
701 "Only iovs overlapping, even partially, with this range will be shown.")
702 iovfilter.add_arguments(
"payloads")
706 if not iovfilter.check_arguments():
710 if db.get_globalTagInfo(args.tag)
is None:
711 B2ERROR(f
"Globaltag '{args.tag}' doesn't exist.")
714 run_range_str = f
' valid in {tuple(args.run_range)}' if args.run_range
else ''
715 args.run_range = IntervalOfValidity(args.run_range)
if args.run_range
else None
717 if args.run
is not None:
718 msg = f
"Obtaining list of iovs for globaltag {args.tag}, exp={args.run[0]}, run={args.run[1]}{iovfilter}"
719 req = db.request(
"GET",
"/iovPayloads", msg, params={
'gtName': args.tag,
'expNumber': args.run[0],
720 'runNumber': args.run[1]})
722 msg = f
"Obtaining list of iovs for globaltag {args.tag}{iovfilter}{run_range_str}"
723 req = db.request(
"GET", f
"/globalTag/{encode_name(args.tag)}/globalTagPayloads", msg)
725 with Pager(f
"List of IoVs{iovfilter}{run_range_str}{' (detailed)' if args.detail else ''}",
True):
727 for item
in req.json():
728 payload = item[
"payload" if 'payload' in item
else "payloadId"]
729 if "payloadIov" in item:
730 iovs = [item[
'payloadIov']]
732 iovs = item[
'payloadIovs']
734 if not iovfilter.check(payload[
'basf2Module'][
'name']):
738 if args.run_range
is not None:
739 if IntervalOfValidity(
740 iov[
'expStart'], iov[
'runStart'], iov[
'expEnd'], iov[
'runEnd']
741 ).intersect(args.run_range)
is None:
747 iov_created = parse_date(iov[
"dtmIns"])
748 iov_modified = parse_date(iov[
"dtmMod"])
749 payload_created = parse_date(payload[
"dtmIns"])
750 payload_modified = parse_date(payload[
"dtmMod"])
752 [
"IoV Id", str(iov[
"payloadIovId"])],
753 [
"first experiment", iov[
"expStart"]],
754 [
"first run", iov[
"runStart"]],
755 [
"final experiment", iov[
"expEnd"]],
756 [
"final run", iov[
"runEnd"]],
757 [
"IoV created", iov_created.astimezone(tz=
None).strftime(
"%Y-%m-%d %H:%M:%S local time")],
758 [
"IoV modified", iov_modified.astimezone(tz=
None).strftime(
"%Y-%m-%d %H:%M:%S local time")],
759 [
"IoV modified by", iov[
"modifiedBy"]],
760 [
"payload Id", str(payload[
"payloadId"])],
761 [
"name", payload[
"basf2Module"][
"name"]],
762 [
"revision", payload[
"revision"]],
763 [
"checksum", payload[
"checksum"]],
764 [
"payloadUrl", payload[
"payloadUrl"]],
765 [
"baseUrl", payload.get(
"baseUrl",
"")],
767 [
"payload created", payload_created.astimezone(tz=
None).strftime(
"%Y-%m-%d %H:%M:%S local time")],
768 [
"payload modified", payload_modified.astimezone(tz=
None).strftime(
"%Y-%m-%d %H:%M:%S local time")],
769 [
"payload modified by", escape_ctrl_chars(payload[
"modifiedBy"])],
772 pretty_print_table(result, [-40,
'*'],
True)
774 payloads.append(PayloadInformation.from_json(payload, iov))
777 def add_ids(table, columns, payloads):
778 """Add the numerical ids to the table"""
780 table[0] += [
"IovId",
"PayloadId"]
782 for row, p
in zip(table[1:], payloads):
783 row += [p.iov_id, p.payload_id]
785 if args.human_readable:
786 table = [[
"Name",
"Rev" if not args.checksums
else "Checksum",
"IoV"]]
787 columns = [
"+", -8
if not args.checksums
else -32, -32]
788 table += [[p.name, p.revision
if not args.checksums
else p.checksum, p.readable_iov()]
for p
in payloads]
789 add_ids(table, columns, payloads)
791 remove_repeated_values(table, columns=[0, 1] + ([-1]
if args.show_ids
else []))
794 table = [[
"Name",
"Rev" if not args.checksums
else "Checksum",
"First Exp",
"First Run",
"Final Exp",
"Final Run"]]
795 table += [[p.name, p.revision
if not args.checksums
else p.checksum] + list(p.iov)
for p
in payloads]
796 columns = [
"+", -8
if not args.checksums
else -32, 6, 6, 6, 6]
797 add_ids(table, columns, payloads)
799 pretty_print_table(table, columns)
802def command_dump(args, db):
804 Dump the content of a given payload
806 .. note:: Version added: release-03-00-00
808 This command will dump the payload contents stored in a given payload. One
809 can either specify the ``payloadId`` (from a previous output of
810 ``b2conditionsdb iov``), the payload name and its revision in the central
811 database, or directly specify a local database payload file.
815 Dump the content of a previously downloaded payload file::
817 $ b2conditionsdb dump -f centraldb/dbstore_BeamParameters_rev_59449.root
819 Dump the content of a payload by name and revision directly from the central database::
821 $ b2conditionsdb dump -r BeamParameters 59449
823 Dump the content of the payload by name which is valid in a given globaltag
824 for a given experiment and run::
826 $ b2conditionsdb dump -g BeamParameters main_2021-08-04 0 0
828 Or directly by payload id from a previous call to ``b2conditionsdb iov``::
830 $ b2conditionsdb dump -i 59685
834 Depending on whether you want to display a payload by its id in the
835 database, its name and revision in the database or from a local file
836 provide **one** of the arguments ``-i``, ``-r``, ``-f`` or ``-g``
838 .. note:: Version changed: after release-04-00-00
840 added argument ``-r`` to directly dump a payload valid for a given run
844 group = args.add_mutually_exclusive_group(required=
True)
845 choice = group.add_mutually_exclusive_group()
846 choice.add_argument(
"-i",
"--id", metavar=
"PAYLOADID", help=
"payload id to dump")
847 choice.add_argument(
"-r",
"--revision", metavar=(
"NAME",
"REVISION"), nargs=2,
848 help=
"Name and revision of the payload to dump")
849 choice.add_argument(
"-f",
"--file", metavar=
"FILENAME", help=
"Dump local payload file")
850 choice.add_argument(
"-g",
"--valid", metavar=(
"NAME",
"GLOBALTAG",
"EXP",
"RUN"), nargs=4,
851 help=
"Dump the payload valid for the given exp, run number in the given globaltag")
852 args.add_argument(
"--show-typenames", default=
False, action=
"store_true",
853 help=
"If given show the type names of all classes. "
854 "This makes output more crowded but can be helpful for complex objects.")
855 args.add_argument(
"--show-streamerinfo", default=
False, action=
"store_true",
856 help=
"If given show the StreamerInfo for the classes in the the payload file. "
857 "This can be helpful to find out which version of a payload object "
858 "is included and what are the members")
866 if not os.path.isfile(filename):
867 B2ERROR(f
"Payloadfile {filename} could not be found")
870 match = re.match(
r"^dbstore_(.*)_rev_(.*).root$", os.path.basename(filename))
872 match = re.match(
r"^(.*)_r(.*).root$", os.path.basename(filename))
874 B2ERROR(
"Filename doesn't follow database convention.\n"
875 "Should be 'dbstore_${payloadname}_rev_${revision}.root' or '${payloadname}_r${revision.root}'")
877 name = match.group(1)
878 revision = match.group(2)
879 payloadId =
"Unknown"
884 req = db.request(
"GET", f
"/payload/{args.id}",
"Getting payload info")
885 payload = PayloadInformation.from_json(req.json())
888 name, rev = args.revision
890 req = db.request(
"GET", f
"/module/{encode_name(name)}/payloads",
"Getting payload info")
892 if p[
"revision"] == rev:
893 payload = PayloadInformation.from_json(p)
896 B2ERROR(f
"Cannot find payload {name} with revision {rev}")
899 name, globaltag, exp, run = args.valid
901 for p
in db.get_all_iovs(globaltag, exp, run, f
", name={name}"):
902 if p.name == name
and (payload
is None or p.revision > payload.revision):
906 B2ERROR(f
"Cannot find payload {name} in globaltag {globaltag} for exp,run {exp},{run}")
909 filename = payload.url
910 revision = payload.revision
911 payloadId = payload.payload_id
915 from ROOT
import TFile, TBufferJSON, cout
918 tfile = TFile.Open(filename)
921 if not tfile
or not tfile.IsOpen():
923 contents = db._session.get(filename, stream=
True)
924 if contents.status_code != requests.codes.ok:
925 B2ERROR(f
"Could not open payload file {filename}")
927 raw_contents = contents.raw.read().decode()
929 obj = tfile.Get(name)
931 json_str = TBufferJSON.ConvertToJSON(obj)
935 Drop some members from ROOT json output.
937 We do not care about fBits, fUniqueID or the typename of sub classes,
938 we assume users are only interested in the data stored in the member
941 obj.pop(
"fBits",
None)
942 obj.pop(
"fUniqueID",
None)
943 if not args.show_typenames:
944 obj.pop(
"_typename",
None)
947 with Pager(f
"Contents of Payload {name}, revision {revision} (id {payloadId})",
True):
948 if args.show_streamerinfo
and tfile:
949 B2INFO(
"StreamerInfo of Payload {name}, revision {revision} (id {payloadId})")
950 tfile.ShowStreamerInfo()
957 if json_str
is not None:
958 B2INFO(f
"Contents of Payload {name}, revision {revision} (id {payloadId})")
961 obj = json.loads(json_str.Data(), object_hook=drop_fbits)
963 pprint.pprint(obj, compact=
True, width=shutil.get_terminal_size((80, 20))[0])
965 B2INFO(f
"Raw contents of Payload {name}, revision {revision} (id {payloadId})")
966 print(escape_ctrl_chars(raw_contents))
968 B2INFO(f
"ROOT contents of Payload {name}, revision {revision} (id {payloadId})")
969 B2WARNING(
"The payload is a valid ROOT file but doesn't contain a payload object with the expected name. "
970 " Automated display of file contents are not possible, showing just entries in the ROOT file.")
975 """Class to recursively show help for an ArgumentParser and all it's sub_parsers"""
978 """Print help message for given parser and call again for all sub parsers"""
980 subparsers_actions = [
981 action
for action
in parser._actions
982 if isinstance(action, argparse._SubParsersAction)]
985 for subparsers_action
in subparsers_actions:
987 for choice, subparser
in subparsers_action.choices.items():
989 print(f
"Command '{prefix}{choice}'")
990 print(subparser.format_help())
994 def __call__(self, parser, namespace, values, option_string=None):
995 """Show full help message"""
997 with Pager(f
"{parser.prog} {option_string}"):
1003def get_argument_parser():
1005 Build a parser with all arguments of all commands
1008 options = argparse.ArgumentParser(add_help=
False)
1009 options.add_argument(
"--debugging", action=
"store_true",
1010 help=
"Enable debugging of http traffic")
1011 options.add_argument(
"--help-full", action=FullHelpAction,
1012 help=
"show help message for all commands and exit")
1013 options.add_argument(
"--base-url", default=
None,
1014 help=
"URI for the base of the REST API, if not given a list of default locations is tried")
1015 options.add_argument(
"--auth-token", type=str, default=
None,
1016 help=
"JSON Web Token necessary for authenticating to the conditions database. "
1017 "Useful only for debugging, since by default the tool automatically "
1018 "gets a token for you by asking the B2MMS username and password. "
1019 "If the environment variable ``$BELLE2_CDB_AUTH_TOKEN`` points to a file with a valid "
1020 "token, such token is used (useful for automatic workflows).")
1022 parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, parents=[options])
1023 parser.set_defaults(func=
lambda x, y: parser.print_help())
1024 parsers = parser.add_subparsers(
1025 title=
"Top level commands",
1026 description=
"To get additional help, run '%(prog)s COMMAND --help'"
1031 for name, func
in sorted(globals().items()):
1032 if not name.startswith(
"command_"):
1037 parts = name.split(
'_')[1:]
1042 if (len(parts) > 1):
1043 parent_parser, parent = subparsers[tuple(parts[:-1])]
1047 parent = parent_parser.add_subparsers(
1048 title=
"sub commands",
1049 description=
"To get additional help, run '%(prog)s COMMAND --help'"
1051 subparsers[tuple(parts[:-1])][1] = parent
1056 helptxt, description = textwrap.dedent(func.__doc__).split(
"\n\n", 1)
1057 command_parser = parent.add_parser(parts[-1], help=helptxt, add_help=
True, description=description,
1058 parents=[options], formatter_class=argparse.RawDescriptionHelpFormatter)
1061 func(command_parser,
None)
1063 command_parser.set_defaults(func=func)
1065 subparsers[tuple(parts)] = [command_parser,
None]
1070def create_symlinks(base):
1071 """Create symlinks from base to all subcommands.
1073 e.g. if the base is ``b2conditionsdb`` then this command will create symlinks
1074 like ``b2conditionsdb-tag-show`` in the same directory
1076 When adding a new command to b2conditionsdb this function needs to be executed
1077 in the framework tools directory
1079 python3 -c 'from conditions_db import cli_main; cli_main.create_symlinks("b2conditionsdb")'
1085 for name
in sorted(globals().keys()):
1086 if not name.startswith(
"command_"):
1088 parts = name.split(
"_")[1:]
1089 if parts
in excluded:
1091 dest = base +
"-".join([
""] + parts)
1095 except FileNotFoundError:
1097 print(f
"create symlink {dest}")
1098 os.symlink(base, dest)
1103 Main function for the command line interface.
1105 it will automatically create an ArgumentParser including all functions which
1106 start with command_ in the global namespace as sub commands. These
1107 functions should take the arguments as first argument and an instance of the
1108 ConditionsDB interface as second argument. If the db interface is None the
1109 first argument is an instance of argparse.ArgumentParser an in this case the
1110 function should just add all needed arguments to the argument parser and
1115 logging.enable_summary(
False)
1117 logging.enable_python_logging =
True
1119 for level
in LogLevel.values.values():
1120 logging.set_info(level, LogInfo.LEVEL | LogInfo.MESSAGE)
1124 sys.argv[0:1] = os.path.basename(sys.argv[0]).split(
'-')
1127 parser = get_argument_parser()
1131 args = parser.parse_args()
1138 nprocess = getattr(args,
"nprocess", 1)
1139 retries = getattr(args,
"retries", 0)
1142 B2WARNING(
"-j must be larger than zero, ignoring")
1143 args.nprocess = nprocess = 1
1145 conditions_db =
ConditionsDB(args.base_url, nprocess, retries)
1148 return args.func(args, conditions_db)
__call__(self, parser, namespace, values, option_string=None)
print_subparsers(self, parser, prefix="")