Belle II Software development
cli_main.py
1#!/usr/bin/env python3
2
3
10
11"""
12This tool provides a command line interface to all the tasks related to the
13:ref:`Conditions database <conditionsdb_overview>`: manage globaltags and iovs
14as well as upload new payloads or download of existing payloads.
15
16The usage of this tool is similar to git: there are sub commands like for
17example ``tag`` which groups all actions related to the management of
18globaltags. All the available commands are listed below.
19
20While the read access to the conditions database is always allowed (e.g.
21for downloading globaltags and payloads), users need a valid JSON Web Token
22(JWT) to authenticate to the conditions database when using this tool for
23creating/manpipulating globaltags or uploading payloads. For practical purposes,
24it is only necessary to know that a JWT is a string containing encrypted
25information, and that string is stored in a file. More information about what
26a JWT is can be found on
27`Wikipedia <https://en.wikipedia.org/wiki/JSON_Web_Token>`_.
28
29.. warning::
30
31 By default, users can only create globaltags whose name starts with
32 ``user_<username>_`` or ``temp_<username>_``, where ``username`` is the
33 B2MMS username.
34
35The tool automatically queries the JWT issuing server
36(https://token.belle2.org) and gets a valid token by asking the B2MMS username
37and password. The issued "default" JWT has a validity of 1 hour; after it
38expires, a new JWT needs to be obtained for authenticating the conditions
39database. When retrieved by this tool, the JWT is stored locally in the file
40``${HOME}/b2cdb_${BELLE2_USER}.token``.
41
42Some selected users (e.g. the calibration managers) are granted a JWT with an
43extended validity (30 days) to allow smooth operations with some automated
44workflows. Such "extended" JWTs are issued by a different server
45(https://token.belle2.org/extended). B2MMS username and password are necessary
46for getting the extended JWT. The extended JWT differs from the default JWT
47only by its validity and can be obtained only by manually querying the
48alternative server. If queried via web browser, a file containing the extended
49JWT will be downloaded in case the user is granted it. The server can also be
50queried via command line using
51``wget --user USERNAME --ask-password --no-check-certificate https://token.belle2.org/extended``
52or ``curl -u USERNAME -k https://token.belle2.org/extended``.
53
54If the environment variable ``${BELLE2_CDB_AUTH_TOKEN}`` is defined and points
55to a file containing a valid JWT, the ``b2conditionsdb`` tools use this token
56to authenticate with the CDB instead of querying the issuing server. This is
57useful when using an extended token. Please note that, in case the JWT to which
58``${BELLE2_CDB_AUTH_TOKEN}`` points is not valid anymore, the
59``b2conditionsdb`` tools will attempt to get a new one and store it into
60``${BELLE2_CDB_AUTH_TOKEN}``. If a new extended token is needed, it has to be
61manually obtained via https://token.belle2.org/extended and stored into
62``${BELLE2_CDB_AUTH_TOKEN}``.
63"""
64
65# Creation of command line parser is done semi-automatically: it looks for
66# functions which are called command_* and will take the * as sub command name
67# and the docstring as documentation: the first line as brief explanation and
68# the remainder as full documentation. command_foo_bar will create subcommand
69# bar inside command foo. The function will be called first with an instance of
70# argument parser where the command line options required for that command
71# should be added. If the command is run it will be called with the parsed
72# arguments first and an instance to the database object as second argument.
73
74# Some of the commands are separated out in separate modules named cli_*.py
75
76import os
77import re
78import sys
79import argparse
80import textwrap
81import json
82import difflib
83import shutil
84import pprint
85import requests
86from basf2 import B2ERROR, B2WARNING, B2INFO, LogLevel, LogInfo, logging, \
87 LogPythonInterface
88from basf2.utils import pretty_print_table
89from terminal_utils import Pager
90from dateutil.parser import parse as parse_date
91from getpass import getuser
92from conditions_db import ConditionsDB, enable_debugging, encode_name, PayloadInformation, set_cdb_authentication_token
93from conditions_db.cli_utils import ItemFilter
94from conditions_db.iov import IntervalOfValidity
95# the command_* functions are imported but not used so disable warning about
96# this if pylama/pylint is used to check
97from conditions_db.cli_upload import command_upload # noqa
98from conditions_db.cli_download import command_download, command_legacydownload # noqa
99from conditions_db.cli_management import command_tag_merge, command_tag_runningupdate, command_iovs, command_iovs_delete, command_iovs_copy, command_iovs_modify # noqa
100
101
102def escape_ctrl_chars(name):
103 """Remove ANSI control characters from strings"""
104 # compile the regex on first use and remember it
105 if not hasattr(escape_ctrl_chars, "_regex"):
106 escape_ctrl_chars._regex = re.compile("[\x00-\x1f\x7f-\x9f]")
107
108 # escape the control characters by putting them in as \xFF
109 def escape(match):
110 if match.group(0).isspace():
111 return match.group(0)
112 return f"\\x{ord(match.group(0)):02x}"
113
114 return escape_ctrl_chars._regex.sub(escape, name)
115
116
117def command_tag(args, db=None):
118 """
119 List, show, create, modify or clone globaltags.
120
121 This command allows to list, show, create modify or clone globaltags in the
122 central database. If no other command is given it will list all tags as if
123 ``%(prog)s show`` was given.
124 """
125
126 # no arguments to define, just a group command
127 if db is not None:
128 # normal call, in this case we just divert to list all tags
129 command_tag_list(args, db)
130
131
132def command_tag_list(args, db=None):
133 """
134 List all available globaltags.
135
136 This command allows to list all globaltags, optionally limiting the output
137 to ones matching a given search term. By default invalidated globaltags
138 will not be included in the list, to show them as well please add
139 ``--with-invalid`` as option. Alternatively one can use ``--only-published``
140 to show only tags which have been published
141
142 If the ``--regex`` option is supplied the search term will be interpreted
143 as a Python regular expression where the case is ignored.
144 """
145
146 tagfilter = ItemFilter(args)
147 if db is None:
148 # db object is none means we should just define arguments
149 args.add_argument("--detail", action="store_true", default=False,
150 help="show detailed information for all tags instead "
151 "of summary table")
152 group = args.add_mutually_exclusive_group()
153 group.add_argument("--with-invalid", action="store_true", default=False,
154 help="if given also invalidated tags will be shown")
155 group.add_argument("--only-published", action="store_true", default=False,
156 help="if given only published tags will be shown")
157 tagfilter.add_arguments("tags")
158 return
159
160 # see if the -f/-e/-r flags are ok
161 if not tagfilter.check_arguments():
162 return 1
163
164 req = db.request("GET", "/globalTags", f"Getting list of globaltags{tagfilter}")
165
166 # now let's filter the tags
167 taglist = []
168 for item in req.json():
169 if not tagfilter.check(item["name"]):
170 continue
171 tagstatus = item["globalTagStatus"]["name"]
172 if not getattr(args, "with_invalid", False) and tagstatus == "INVALID":
173 continue
174 if getattr(args, "only_published", False) and tagstatus != "PUBLISHED":
175 continue
176 taglist.append(item)
177
178 # done, sort by name
179 taglist.sort(key=lambda x: x["name"])
180
181 # and print, either detailed info for each tag or summary table at the end
182 table = []
183 with Pager(f"List of globaltags{tagfilter}{' (detailed)' if getattr(args, 'detail', False) else ''}", True):
184 for item in taglist:
185 if getattr(args, "detail", False):
186 print_globaltag(db, item)
187 else:
188 table.append([
189 item["globalTagId"],
190 item["name"],
191 escape_ctrl_chars(item.get("description", "")),
192 item["globalTagType"]["name"],
193 item["globalTagStatus"]["name"],
194 item["payloadCount"],
195 ])
196
197 if not getattr(args, "detail", False):
198 table.insert(0, ["id", "name", "description", "type", "status", "# payloads"])
199 pretty_print_table(table, [-10, 0, "*", -10, -10, -10], min_flexible_width=32)
200
201
202def print_globaltag(db, *tags):
203 """ Print detailed globaltag information for the given globaltags side by side"""
204 results = [["id"], ["name"], ["description"], ["type"], ["status"],
205 ["# payloads"], ["created"], ["modified"], ["modified by"]]
206 for info in tags:
207 if info is None:
208 continue
209
210 if isinstance(info, str):
211 try:
212 req = db.request("GET", f"/globalTag/{encode_name(info)}",
213 f"Getting info for globaltag {info}")
214 except ConditionsDB.RequestError as e:
215 # ok, there's an error for this one, let's continue with the other
216 # ones
217 B2ERROR(str(e))
218 continue
219
220 info = req.json()
221
222 created = parse_date(info["dtmIns"])
223 modified = parse_date(info["dtmMod"])
224 results[0].append(str(info["globalTagId"])),
225 results[1].append(info["name"]),
226 results[2].append(escape_ctrl_chars(info.get("description", "")))
227 results[3].append(info["globalTagType"]["name"])
228 results[4].append(info["globalTagStatus"]["name"]),
229 results[5].append(info["payloadCount"]),
230 results[6].append(created.astimezone(tz=None).strftime("%Y-%m-%d %H:%M:%S local time"))
231 results[7].append(modified.astimezone(tz=None).strftime("%Y-%m-%d %H:%M:%S local time"))
232 results[8].append(escape_ctrl_chars(info["modifiedBy"]))
233
234 ntags = len(results[0]) - 1
235 if ntags > 0:
236 pretty_print_table(results, [11] + ['*'] * ntags, True)
237 return ntags
238
239
240def change_state(db, tag, state, force=False):
241 """Change the state of a global tag
242
243 If the new state is not revertible then ask for confirmation
244 """
245 state = state.upper()
246 if state in ["INVALID", "PUBLISHED"] and not force:
247 name = input(f"ATTENTION: Marking a tag as {state} cannot be undone.\n"
248 "If you are sure you want to continue it please enter the tag name again: ")
249 if name != tag:
250 B2ERROR("Names don't match, aborting")
251 return 1
252
253 db.request("PUT", f"/globalTag/{encode_name(tag)}/updateGlobalTagStatus/{state}",
254 f"Changing globaltag state {tag} to {state}")
255
256
257def command_tag_show(args, db=None):
258 """
259 Show details about globaltags
260
261 This command will show details for the given globaltags like name,
262 description and number of payloads.
263 """
264
265 # this one is a bit similar to command_tag_list but gets single tag
266 # information from the database and thus no need for filtering. It will
267 # always show the detailed information
268
269 if db is None:
270 args.add_argument("tag", metavar="TAGNAME", nargs="+", help="globaltags to show")
271 return
272
273 # we retrieved all we could, print them
274 ntags = 0
275 with Pager("globaltag Information", True):
276 for tag in args.tag:
277 ntags += print_globaltag(db, tag)
278
279 # return the number of tags which could not get retrieved
280 return len(args.tag) - ntags
281
282
283def command_tag_create(args, db=None):
284 """
285 Create a new globaltag
286
287 This command creates a new globaltag in the database with the given name
288 and description. The name can only contain alphanumeric characters and the
289 characters ``+-_:``.
290
291 .. warning::
292
293 By default, users can only create globaltags whose name starts with
294 ``user_<username>_`` or ``temp_<username>_``, where ``username`` is the
295 B2MMS username.
296 """
297
298 if db is None:
299 args.add_argument("type", metavar="TYPE", help="type of the globaltag to create, usually one of DEV or RELEASE")
300 args.add_argument("tag", metavar="TAGNAME", help="name of the globaltag to create")
301 args.add_argument("description", metavar="DESCRIPTION", help="description of the globaltag")
302 args.add_argument("-u", "--user", metavar="USER", help="username who created the tag. "
303 "If not given we will try to supply a useful default")
304 return
305
306 set_cdb_authentication_token(db, args.auth_token)
307
308 # create tag info needed for creation
309 info = {"name": args.tag, "description": args.description, "modifiedBy": args.user, "isDefault": False}
310 # add user information if not given by command line
311 if args.user is None:
312 info["modifiedBy"] = os.environ.get("BELLE2_USER", getuser())
313
314 typeinfo = db.get_globalTagType(args.type)
315 if typeinfo is None:
316 return 1
317
318 req = db.request("POST", f"/globalTag/{encode_name(typeinfo['name'])}",
319 "Creating globaltag {name}".format(**info),
320 json=info)
321 B2INFO("Successfully created globaltag {name} (id={globalTagId})".format(**req.json()))
322
323
324def command_tag_modify(args, db=None):
325 """
326 Modify a globaltag by changing name or description
327
328 This command allows to change the name or description of an existing globaltag.
329 You can supply any combination of -n,-d,-t and only the given values will be changed
330 """
331 if db is None:
332 args.add_argument("tag", metavar="TAGNAME", help="globaltag to modify")
333 args.add_argument("-n", "--name", help="new name")
334 args.add_argument("-d", "--description", help="new description")
335 args.add_argument("-t", "--type", help="new type of the globaltag")
336 args.add_argument("-u", "--user", metavar="USER", help="username who created the tag. "
337 "If not given we will try to supply a useful default")
338 args.add_argument("-s", "--state", help="new globaltag state, see the command ``tag state`` for details")
339 return
340
341 set_cdb_authentication_token(db, args.auth_token)
342
343 # first we need to get the old tag information
344 req = db.request("GET", f"/globalTag/{encode_name(args.tag)}",
345 f"Getting info for globaltag {args.tag}")
346
347 # now we update the tag information
348 info = req.json()
349 old_name = info["name"]
350 changed = False
351 for key in ["name", "description"]:
352 value = getattr(args, key)
353 if value is not None and value != info[key]:
354 info[key] = value
355 changed = True
356
357 info["modifiedBy"] = os.environ.get("BELLE2_USER", os.getlogin()) if args.user is None else args.user
358
359 if args.type is not None:
360 # for the type we need to know which types are defined
361 typeinfo = db.get_globalTagType(args.type)
362 if typeinfo is None:
363 return 1
364 # seems so, ok modify the tag info
365 if info['globalTagType'] != typeinfo:
366 info["globalTagType"] = typeinfo
367 changed = True
368
369 # and push the changed info to the server
370 if changed:
371 db.request("PUT", "/globalTag",
372 "Modifying globaltag {} (id={globalTagId})".format(old_name, **info),
373 json=info)
374
375 if args.state is not None:
376 name = args.name if args.name is not None else old_name
377 return change_state(db, name, args.state)
378
379
380def command_tag_clone(args, db=None):
381 """
382 Clone a given globaltag including all IoVs
383
384 This command allows to clone a given globaltag with a new name but still
385 containing all the IoVs defined in the original globaltag.
386 """
387
388 if db is None:
389 args.add_argument("tag", metavar="TAGNAME", help="globaltag to be cloned")
390 args.add_argument("name", metavar="NEWNAME", help="name of the cloned globaltag")
391 return
392
393 set_cdb_authentication_token(db, args.auth_token)
394
395 # first we need to get the old tag information
396 req = db.request("GET", f"/globalTag/{encode_name(args.tag)}",
397 f"Getting info for globaltag {args.tag}")
398 info = req.json()
399
400 # now we clone the tag. id came from the database so no need for escape
401 req = db.request("POST", "/globalTags/{globalTagId}".format(**info),
402 "Cloning globaltag {name} (id={globalTagId})".format(**info))
403
404 # it gets a stupid name "{ID}_copy_of_{name}" so we change it to something
405 # nice like the user asked
406 cloned_info = req.json()
407 cloned_info["name"] = args.name
408 # and push the changed info to the server
409 db.request("PUT", "/globalTag", "Renaming globaltag {name} (id={globalTagId})".format(**cloned_info),
410 json=cloned_info)
411
412
413def command_tag_state(args, db):
414 """
415 Change the state of a globaltag.
416
417 This command changes the state of a globaltag to the given value.
418
419 Usually the valid states are
420
421 OPEN
422 Tag can be modified, payloads and iovs can be created and deleted. This
423 is the default state for new or cloned globaltags and is not suitable
424 for use in data analysis
425
426 Can be transitioned to TESTING, RUNNING
427
428 TESTING
429 Tag cannot be modified and is suitable for testing but can be reopened
430
431 Can be transitioned to VALIDATED, OPEN
432
433 VALIDATED
434 Tag cannot be modified and has been tested.
435
436 Can be transitioned to PUBLISHED, OPEN
437
438 PUBLISHED
439 Tag cannot be modified and is suitable for user analysis
440
441 Can only be transitioned to INVALID
442
443 RUNNING
444 Tag can only be modified by adding new runs, not modifying the payloads
445 for existing runs.
446
447 INVALID:
448 Tag is invalid and should not be used for anything.
449
450 This state is end of life for a globaltag and cannot be transitioned to
451 any other state.
452
453 .. note:: Version added: release-04-00-00
454 """
455 if db is None:
456 args.add_argument("tag", metavar="TAGNAME", help="globaltag to be changed")
457 args.add_argument("state", metavar="STATE", help="new state for the globaltag")
458 args.add_argument("--force", default=False, action="store_true", help=argparse.SUPPRESS)
459 return
460
461 set_cdb_authentication_token(db, args.auth_token)
462
463 return change_state(db, args.tag, args.state, args.force)
464
465
466def remove_repeated_values(table, columns, keep=None):
467 """Strip repeated values from a table of values
468
469 This function takes a table (a list of lists with all the same length) and
470 removes values in certain columns if they are identical in consecutive rows.
471
472 It does this in a dependent way only if the previous columns are identical
473 will it continue stripping further columns. For example, given the table ::
474
475 table = [
476 ["A", "a"],
477 ["B", "a"],
478 ["B", "a"],
479 ["B", "b"],
480 ]
481
482 If we want to remove duplicates in all columns in order it would look like this:
483
484 >>> remove_repeated_values(table, [0,1])
485 [
486 ["A", "a"],
487 ["B", "a"],
488 [ "", ""],
489 [ "", "b"],
490 ]
491
492 But we can give selected columns to strip one after the other
493
494 >>> remove_repeated_values(table, [1,0])
495 [
496 ["A", "a"],
497 ["B", ""],
498 [ "", ""],
499 ["B", "b"],
500 ]
501
502 In addition, we might want to only strip some columns if previous columns
503 were identical but keep the values of the previous column. For this one can
504 supply ``keep``:
505
506 >>> remove_repeated_values(table, [0,1,2], keep=[0])
507 [
508 ["A", "a"],
509 ["B", "a"],
510 ["B", ""],
511 ["B", "b"],
512 ]
513
514 Parameters:
515 table (list(list(str))): 2d table of values
516 columns (list(int)): indices of columns to consider in order
517 keep (set(int)): indices of columns to not strip
518 """
519 last_values = [None] * len(columns)
520 for row in table[1:]:
521 current_values = [row[i] for i in columns]
522 for i, curr, last in zip(columns, current_values, last_values):
523 if curr != last:
524 break
525
526 if keep and i in keep:
527 continue
528
529 row[i] = ""
530
531 last_values = current_values
532
533
534def command_diff(args, db):
535 """
536 Compare two globaltags
537
538 This command allows to compare two globaltags. It will show the changes in
539 a format similar to a unified diff but by default it will not show any
540 context, only the new or removed payloads. Added payloads are marked with a
541 ``+`` in the first column, removed payloads with a ``-``
542
543 If ``--full`` is given it will show all payloads, even the ones common to
544 both globaltags. The differences can be limited to a given run and
545 limited to a set of payloads names using ``--filter`` or ``--exclude``. If
546 the ``--regex`` option is supplied the search term will be interpreted as a
547 python regular expression where the case is ignored.
548
549 .. note:: Version changed: release-03-00-00
550
551 modified output structure and added ``--human-readable``
552 .. note:: Version changed: after release-04-00-00
553
554 added parameter ``--checksums`` and ``--show-ids``
555 """
556 iovfilter = ItemFilter(args)
557 if db is None:
558 args.add_argument("--full", default=False, action="store_true",
559 help="If given print all iovs, also those which are the same in both tags")
560 args.add_argument("--run", type=int, nargs=2, metavar="N", help="exp and run numbers "
561 "to limit showing iovs to a ones present in a given run")
562 args.add_argument("--human-readable", default=False, action="store_true",
563 help="If given the iovs will be written in a more human friendly format. "
564 "Also repeated payload names will be omitted to create a more readable listing.")
565 args.add_argument("--checksums", default=False, action="store_true",
566 help="If given don't show the revision number but the md5 checksum")
567 args.add_argument("--show-ids", default=False, action="store_true",
568 help="If given also show the payload and iov ids for each iov")
569
570 args.add_argument("tagA", metavar="TAGNAME1", help="base for comparison")
571 args.add_argument("tagB", metavar="TAGNAME2", help="tagname to compare")
572 args.add_argument("--run-range", nargs=4, default=None, type=int,
573 metavar=("FIRST_EXP", "FIRST_RUN", "FINAL_EXP", "FINAL_RUN"),
574 help="Can be four numbers to limit the run range to be compared"
575 "Only iovs overlapping, even partially, with this range will be considered.")
576 iovfilter.add_arguments("payloads")
577 return
578
579 # check arguments
580 if not iovfilter.check_arguments():
581 return 1
582
583 with Pager(f"Differences between globaltags {args.tagA} and {args.tagB}{iovfilter}", True):
584 print("globaltags to be compared:")
585 ntags = print_globaltag(db, args.tagA, args.tagB)
586 if ntags != 2:
587 return 1
588 print()
589 listA = [
590 e for e in db.get_all_iovs(
591 args.tagA,
592 message=str(iovfilter),
593 run_range=args.run_range) if iovfilter.check(
594 e.name)]
595 listB = [
596 e for e in db.get_all_iovs(
597 args.tagB,
598 message=str(iovfilter),
599 run_range=args.run_range) if iovfilter.check(
600 e.name)]
601
602 B2INFO("Comparing contents ...")
603 diff = difflib.SequenceMatcher(a=listA, b=listB)
604 table = [["", "Name", "Rev" if not args.checksums else "Checksum"]]
605 columns = [1, "+", -8 if not args.checksums else -32]
606
607 if args.human_readable:
608 table[0] += ["Iov"]
609 columns += [-36]
610 else:
611 table[0] += ["First Exp", "First Run", "Final Exp", "Final Run"]
612 columns += [6, 6, 6, 6]
613
614 if args.show_ids:
615 table[0] += ["IovId", "PayloadId"]
616 columns += [7, 9]
617
618 def add_payloads(opcode, payloads):
619 """Add a list of payloads to the table, filling the first column with opcode"""
620 for p in payloads:
621 row = [opcode, p.name, p.revision if not args.checksums else p.checksum]
622 if args.human_readable:
623 row += [p.readable_iov()]
624 else:
625 row += list(p.iov)
626
627 if args.show_ids:
628 row += [p.iov_id, p.payload_id]
629 table.append(row)
630
631 for tag, i1, i2, j1, j2 in diff.get_opcodes():
632 if tag == "equal":
633 if not args.full:
634 continue
635 add_payloads(" ", listB[j1:j2])
636 if tag in ["delete", "replace"]:
637 add_payloads("-", listA[i1:i2])
638 if tag in ["insert", "replace"]:
639 add_payloads("+", listB[j1:j2])
640
641 if args.human_readable:
642 # strip repeated names, revision, payloadid, to make it more readable.
643 # this is dependent on the fact that the opcode is still the same but we
644 # don't want to strip the opcode ...
645 remove_repeated_values(table, [0, 1, 2] + ([-1] if args.show_ids else []), keep=[0])
646
647 def color_row(row, widths, line):
648 if not LogPythonInterface.terminal_supports_colors():
649 return line
650 begin = {'+': '\x1b[32m', '-': '\x1b[31m'}.get(row[0], "")
651 end = '\x1b[0m'
652 return begin + line + end
653
654 # print the table but make sure the first column is empty except for
655 # added/removed lines so that it can be copy-pasted into a diff syntax
656 # highlighting area (say merge request description)
657 print(f" Differences between {args.tagA} and {args.tagB}")
658 pretty_print_table(table, columns, transform=color_row,
659 hline_formatter=lambda w: " " + (w - 1) * '-')
660
661
662def command_iov(args, db):
663 """
664 List all IoVs defined in a globaltag, optionally limited to a run range
665
666 This command lists all IoVs defined in a given globaltag. The list can be
667 limited to a given run and optionally searched using ``--filter`` or
668 ``--exclude``. If the ``--regex`` option is supplied the search term will
669 be interpreted as a Python regular expression where the case is ignored.
670
671 .. note:: Version changed: release-03-00-00
672
673 modified output structure and added ``--human-readable``
674 .. note:: Version changed: after release-04-00-00
675
676 added parameter ``--checksums`` and ``--show-ids``
677 .. note:: Version changed: after release-08-00-04
678
679 added parameter ``--run-range``
680 """
681
682 iovfilter = ItemFilter(args)
683
684 if db is None:
685 args.add_argument("tag", metavar="TAGNAME", help="globaltag for which the the IoVs should be listed")
686 args.add_argument("--run", type=int, nargs=2, metavar="N", help="exp and run numbers "
687 "to limit showing iovs to a ones present in a given run")
688 args.add_argument("--detail", action="store_true", default=False,
689 help="if given show a detailed information for all "
690 "IoVs including details of the payloads")
691 args.add_argument("--human-readable", default=False, action="store_true",
692 help="If given the iovs will be written in a more human friendly format. "
693 "Also repeated payload names will be omitted to create a more readable listing.")
694 args.add_argument("--checksums", default=False, action="store_true",
695 help="If given don't show the revision number but the md5 checksum")
696 args.add_argument("--show-ids", default=False, action="store_true",
697 help="If given also show the payload and iov ids for each iov")
698 args.add_argument("--run-range", nargs=4, default=None, type=int,
699 metavar=("FIRST_EXP", "FIRST_RUN", "FINAL_EXP", "FINAL_RUN"),
700 help="Can be four numbers to limit the run range to be shown"
701 "Only iovs overlapping, even partially, with this range will be shown.")
702 iovfilter.add_arguments("payloads")
703 return
704
705 # check arguments
706 if not iovfilter.check_arguments():
707 return 1
708
709 # Check if the globaltag exists otherwise I get the same result for an empty global tag or for a non-existing one
710 if db.get_globalTagInfo(args.tag) is None:
711 B2ERROR(f"Globaltag '{args.tag}' doesn't exist.")
712 return False
713
714 run_range_str = f' valid in {tuple(args.run_range)}' if args.run_range else ''
715 args.run_range = IntervalOfValidity(args.run_range) if args.run_range else None
716
717 if args.run is not None:
718 msg = f"Obtaining list of iovs for globaltag {args.tag}, exp={args.run[0]}, run={args.run[1]}{iovfilter}"
719 req = db.request("GET", "/iovPayloads", msg, params={'gtName': args.tag, 'expNumber': args.run[0],
720 'runNumber': args.run[1]})
721 else:
722 msg = f"Obtaining list of iovs for globaltag {args.tag}{iovfilter}{run_range_str}"
723 req = db.request("GET", f"/globalTag/{encode_name(args.tag)}/globalTagPayloads", msg)
724
725 with Pager(f"List of IoVs{iovfilter}{run_range_str}{' (detailed)' if args.detail else ''}", True):
726 payloads = []
727 for item in req.json():
728 payload = item["payload" if 'payload' in item else "payloadId"]
729 if "payloadIov" in item:
730 iovs = [item['payloadIov']]
731 else:
732 iovs = item['payloadIovs']
733
734 if not iovfilter.check(payload['basf2Module']['name']):
735 continue
736
737 for iov in iovs:
738 if args.run_range is not None:
739 if IntervalOfValidity(
740 iov['expStart'], iov['runStart'], iov['expEnd'], iov['runEnd']
741 ).intersect(args.run_range) is None:
742 continue
743
744 if args.detail:
745 # detailed mode, show a table with all information for each
746 # iov
747 iov_created = parse_date(iov["dtmIns"])
748 iov_modified = parse_date(iov["dtmMod"])
749 payload_created = parse_date(payload["dtmIns"])
750 payload_modified = parse_date(payload["dtmMod"])
751 result = [
752 ["IoV Id", str(iov["payloadIovId"])],
753 ["first experiment", iov["expStart"]],
754 ["first run", iov["runStart"]],
755 ["final experiment", iov["expEnd"]],
756 ["final run", iov["runEnd"]],
757 ["IoV created", iov_created.astimezone(tz=None).strftime("%Y-%m-%d %H:%M:%S local time")],
758 ["IoV modified", iov_modified.astimezone(tz=None).strftime("%Y-%m-%d %H:%M:%S local time")],
759 ["IoV modified by", iov["modifiedBy"]],
760 ["payload Id", str(payload["payloadId"])],
761 ["name", payload["basf2Module"]["name"]],
762 ["revision", payload["revision"]],
763 ["checksum", payload["checksum"]],
764 ["payloadUrl", payload["payloadUrl"]],
765 ["baseUrl", payload.get("baseUrl", "")],
766 # print created and modified timestamps in local time zone
767 ["payload created", payload_created.astimezone(tz=None).strftime("%Y-%m-%d %H:%M:%S local time")],
768 ["payload modified", payload_modified.astimezone(tz=None).strftime("%Y-%m-%d %H:%M:%S local time")],
769 ["payload modified by", escape_ctrl_chars(payload["modifiedBy"])],
770 ]
771 print()
772 pretty_print_table(result, [-40, '*'], True)
773 else:
774 payloads.append(PayloadInformation.from_json(payload, iov))
775
776 if not args.detail:
777 def add_ids(table, columns, payloads):
778 """Add the numerical ids to the table"""
779 if args.show_ids:
780 table[0] += ["IovId", "PayloadId"]
781 columns += [9, 9]
782 for row, p in zip(table[1:], payloads):
783 row += [p.iov_id, p.payload_id]
784 payloads.sort()
785 if args.human_readable:
786 table = [["Name", "Rev" if not args.checksums else "Checksum", "IoV"]]
787 columns = ["+", -8 if not args.checksums else -32, -32]
788 table += [[p.name, p.revision if not args.checksums else p.checksum, p.readable_iov()] for p in payloads]
789 add_ids(table, columns, payloads)
790 # strip repeated names, revision, payloadid, to make it more readable
791 remove_repeated_values(table, columns=[0, 1] + ([-1] if args.show_ids else []))
792
793 else:
794 table = [["Name", "Rev" if not args.checksums else "Checksum", "First Exp", "First Run", "Final Exp", "Final Run"]]
795 table += [[p.name, p.revision if not args.checksums else p.checksum] + list(p.iov) for p in payloads]
796 columns = ["+", -8 if not args.checksums else -32, 6, 6, 6, 6]
797 add_ids(table, columns, payloads)
798
799 pretty_print_table(table, columns)
800
801
802def command_dump(args, db):
803 """
804 Dump the content of a given payload
805
806 .. note:: Version added: release-03-00-00
807
808 This command will dump the payload contents stored in a given payload. One
809 can either specify the ``payloadId`` (from a previous output of
810 ``b2conditionsdb iov``), the payload name and its revision in the central
811 database, or directly specify a local database payload file.
812
813 .. rubric:: Examples
814
815 Dump the content of a previously downloaded payload file::
816
817 $ b2conditionsdb dump -f centraldb/dbstore_BeamParameters_rev_59449.root
818
819 Dump the content of a payload by name and revision directly from the central database::
820
821 $ b2conditionsdb dump -r BeamParameters 59449
822
823 Dump the content of the payload by name which is valid in a given globaltag
824 for a given experiment and run::
825
826 $ b2conditionsdb dump -g BeamParameters main_2021-08-04 0 0
827
828 Or directly by payload id from a previous call to ``b2conditionsdb iov``::
829
830 $ b2conditionsdb dump -i 59685
831
832 .. rubric:: Usage
833
834 Depending on whether you want to display a payload by its id in the
835 database, its name and revision in the database or from a local file
836 provide **one** of the arguments ``-i``, ``-r``, ``-f`` or ``-g``
837
838 .. note:: Version changed: after release-04-00-00
839
840 added argument ``-r`` to directly dump a payload valid for a given run
841 in a given globaltag
842 """
843 if db is None:
844 group = args.add_mutually_exclusive_group(required=True)
845 choice = group.add_mutually_exclusive_group()
846 choice.add_argument("-i", "--id", metavar="PAYLOADID", help="payload id to dump")
847 choice.add_argument("-r", "--revision", metavar=("NAME", "REVISION"), nargs=2,
848 help="Name and revision of the payload to dump")
849 choice.add_argument("-f", "--file", metavar="FILENAME", help="Dump local payload file")
850 choice.add_argument("-g", "--valid", metavar=("NAME", "GLOBALTAG", "EXP", "RUN"), nargs=4,
851 help="Dump the payload valid for the given exp, run number in the given globaltag")
852 args.add_argument("--show-typenames", default=False, action="store_true",
853 help="If given show the type names of all classes. "
854 "This makes output more crowded but can be helpful for complex objects.")
855 args.add_argument("--show-streamerinfo", default=False, action="store_true",
856 help="If given show the StreamerInfo for the classes in the the payload file. "
857 "This can be helpful to find out which version of a payload object "
858 "is included and what are the members")
859
860 return
861
862 payload = None
863 # local file, don't query database at all
864 if args.file:
865 filename = args.file
866 if not os.path.isfile(filename):
867 B2ERROR(f"Payloadfile {filename} could not be found")
868 return 1
869
870 match = re.match(r"^dbstore_(.*)_rev_(.*).root$", os.path.basename(filename))
871 if not match:
872 match = re.match(r"^(.*)_r(.*).root$", os.path.basename(filename))
873 if not match:
874 B2ERROR("Filename doesn't follow database convention.\n"
875 "Should be 'dbstore_${payloadname}_rev_${revision}.root' or '${payloadname}_r${revision.root}'")
876 return 1
877 name = match.group(1)
878 revision = match.group(2)
879 payloadId = "Unknown"
880 else:
881 # otherwise do just that: query the database for either payload id or
882 # the name,revision
883 if args.id:
884 req = db.request("GET", f"/payload/{args.id}", "Getting payload info")
885 payload = PayloadInformation.from_json(req.json())
886 name = payload.name
887 elif args.revision:
888 name, rev = args.revision
889 rev = int(rev)
890 req = db.request("GET", f"/module/{encode_name(name)}/payloads", "Getting payload info")
891 for p in req.json():
892 if p["revision"] == rev:
893 payload = PayloadInformation.from_json(p)
894 break
895 else:
896 B2ERROR(f"Cannot find payload {name} with revision {rev}")
897 return 1
898 elif args.valid:
899 name, globaltag, exp, run = args.valid
900 payload = None
901 for p in db.get_all_iovs(globaltag, exp, run, f", name={name}"):
902 if p.name == name and (payload is None or p.revision > payload.revision):
903 payload = p
904
905 if payload is None:
906 B2ERROR(f"Cannot find payload {name} in globaltag {globaltag} for exp,run {exp},{run}")
907 return 1
908
909 filename = payload.url
910 revision = payload.revision
911 payloadId = payload.payload_id
912 del payload
913
914 # late import of ROOT because of all the side effects
915 from ROOT import TFile, TBufferJSON, cout
916
917 # remote http opening or local file
918 tfile = TFile.Open(filename)
919 json_str = None
920 raw_contents = None
921 if not tfile or not tfile.IsOpen():
922 # could be a non-root payload file
923 contents = db._session.get(filename, stream=True)
924 if contents.status_code != requests.codes.ok:
925 B2ERROR(f"Could not open payload file {filename}")
926 return 1
927 raw_contents = contents.raw.read().decode()
928 else:
929 obj = tfile.Get(name)
930 if obj:
931 json_str = TBufferJSON.ConvertToJSON(obj)
932
933 def drop_fbits(obj):
934 """
935 Drop some members from ROOT json output.
936
937 We do not care about fBits, fUniqueID or the typename of sub classes,
938 we assume users are only interested in the data stored in the member
939 variables
940 """
941 obj.pop("fBits", None)
942 obj.pop("fUniqueID", None)
943 if not args.show_typenames:
944 obj.pop("_typename", None)
945 return obj
946
947 with Pager(f"Contents of Payload {name}, revision {revision} (id {payloadId})", True):
948 if args.show_streamerinfo and tfile:
949 B2INFO("StreamerInfo of Payload {name}, revision {revision} (id {payloadId})")
950 tfile.ShowStreamerInfo()
951 # sadly this prints to std::cout or even stdout but doesn't flush ... so we have
952 # to make sure std::cout is flushed before printing anything else
953 cout.flush()
954 # and add a newline
955 print()
956
957 if json_str is not None:
958 B2INFO(f"Contents of Payload {name}, revision {revision} (id {payloadId})")
959 # load the json as python object dropping some things we don't want to
960 # print
961 obj = json.loads(json_str.Data(), object_hook=drop_fbits)
962 # print the object content using pretty print with a certain width
963 pprint.pprint(obj, compact=True, width=shutil.get_terminal_size((80, 20))[0])
964 elif raw_contents:
965 B2INFO(f"Raw contents of Payload {name}, revision {revision} (id {payloadId})")
966 print(escape_ctrl_chars(raw_contents))
967 elif tfile:
968 B2INFO(f"ROOT contents of Payload {name}, revision {revision} (id {payloadId})")
969 B2WARNING("The payload is a valid ROOT file but doesn't contain a payload object with the expected name. "
970 " Automated display of file contents are not possible, showing just entries in the ROOT file.")
971 tfile.ls()
972
973
974class FullHelpAction(argparse._HelpAction):
975 """Class to recursively show help for an ArgumentParser and all it's sub_parsers"""
976
977 def print_subparsers(self, parser, prefix=""):
978 """Print help message for given parser and call again for all sub parsers"""
979 # retrieve subparsers from parser
980 subparsers_actions = [
981 action for action in parser._actions
982 if isinstance(action, argparse._SubParsersAction)]
983 # there will probably only be one subparser_action,
984 # but better save than sorry
985 for subparsers_action in subparsers_actions:
986 # get all subparsers and print help
987 for choice, subparser in subparsers_action.choices.items():
988 print()
989 print(f"Command '{prefix}{choice}'")
990 print(subparser.format_help())
991
992 self.print_subparsers(subparser, prefix=f"{prefix}{choice} ")
993
994 def __call__(self, parser, namespace, values, option_string=None):
995 """Show full help message"""
996 # run in pager because amount of options will be looong
997 with Pager(f"{parser.prog} {option_string}"):
998 parser.print_help()
999 self.print_subparsers(parser)
1000 parser.exit()
1001
1002
1003def get_argument_parser():
1004 """
1005 Build a parser with all arguments of all commands
1006 """
1007 # extra ArgumentParser with the global options just for reusability
1008 options = argparse.ArgumentParser(add_help=False)
1009 options.add_argument("--debugging", action="store_true",
1010 help="Enable debugging of http traffic")
1011 options.add_argument("--help-full", action=FullHelpAction,
1012 help="show help message for all commands and exit")
1013 options.add_argument("--base-url", default=None,
1014 help="URI for the base of the REST API, if not given a list of default locations is tried")
1015 options.add_argument("--auth-token", type=str, default=None,
1016 help="JSON Web Token necessary for authenticating to the conditions database. "
1017 "Useful only for debugging, since by default the tool automatically "
1018 "gets a token for you by asking the B2MMS username and password. "
1019 "If the environment variable ``$BELLE2_CDB_AUTH_TOKEN`` points to a file with a valid "
1020 "token, such token is used (useful for automatic workflows).")
1021
1022 parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, parents=[options])
1023 parser.set_defaults(func=lambda x, y: parser.print_help())
1024 parsers = parser.add_subparsers(
1025 title="Top level commands",
1026 description="To get additional help, run '%(prog)s COMMAND --help'"
1027 )
1028
1029 subparsers = {}
1030 # now we go through all the functions defined which start with command_
1031 for name, func in sorted(globals().items()):
1032 if not name.startswith("command_"):
1033 continue
1034 # we interpret command_foo_bar_baz as subcommand baz of subcommand bar
1035 # of subcommand foo. So let's split this into all commands and remove
1036 # the command_
1037 parts = name.split('_')[1:]
1038 # now we need to get the subparsers instance for the parent command. if
1039 # the command is top level, e.g. foo, we just use parsers. Otherwise we
1040 # go look into the dict of subparsers for command chains.
1041 parent = parsers
1042 if (len(parts) > 1):
1043 parent_parser, parent = subparsers[tuple(parts[:-1])]
1044 # if we are the first subcommand to a given command we have to add
1045 # the subparsers. do that and add it back to the dict
1046 if parent is None:
1047 parent = parent_parser.add_subparsers(
1048 title="sub commands",
1049 description="To get additional help, run '%(prog)s COMMAND --help'"
1050 )
1051 subparsers[tuple(parts[:-1])][1] = parent
1052 # so we have our subparsers instance, now create argument parser for the
1053 # function. We use the first part of the function docstring as help text
1054 # and everything after the first empty line as description of the
1055 # command
1056 helptxt, description = textwrap.dedent(func.__doc__).split("\n\n", 1)
1057 command_parser = parent.add_parser(parts[-1], help=helptxt, add_help=True, description=description,
1058 parents=[options], formatter_class=argparse.RawDescriptionHelpFormatter)
1059 # now call the function with the parser as first argument and no
1060 # database instance. This let's them define their own arguments
1061 func(command_parser, None)
1062 # and set the function as default to be called for later
1063 command_parser.set_defaults(func=func)
1064 # also add it back to the list of subparsers
1065 subparsers[tuple(parts)] = [command_parser, None]
1066
1067 return parser
1068
1069
1070def create_symlinks(base):
1071 """Create symlinks from base to all subcommands.
1072
1073 e.g. if the base is ``b2conditionsdb`` then this command will create symlinks
1074 like ``b2conditionsdb-tag-show`` in the same directory
1075
1076 When adding a new command to b2conditionsdb this function needs to be executed
1077 in the framework tools directory
1078
1079 python3 -c 'from conditions_db import cli_main; cli_main.create_symlinks("b2conditionsdb")'
1080 """
1081 import os
1082 excluded = [
1083 ['tag'] # the tag command without subcommand is not very useful
1084 ]
1085 for name in sorted(globals().keys()):
1086 if not name.startswith("command_"):
1087 continue
1088 parts = name.split("_")[1:]
1089 if parts in excluded:
1090 continue
1091 dest = base + "-".join([""] + parts)
1092
1093 try:
1094 os.remove(dest)
1095 except FileNotFoundError:
1096 pass
1097 print(f"create symlink {dest}")
1098 os.symlink(base, dest)
1099
1100
1101def main():
1102 """
1103 Main function for the command line interface.
1104
1105 it will automatically create an ArgumentParser including all functions which
1106 start with command_ in the global namespace as sub commands. These
1107 functions should take the arguments as first argument and an instance of the
1108 ConditionsDB interface as second argument. If the db interface is None the
1109 first argument is an instance of argparse.ArgumentParser an in this case the
1110 function should just add all needed arguments to the argument parser and
1111 return.
1112 """
1113
1114 # disable error summary
1115 logging.enable_summary(False)
1116 # log via python stdout to be able to capture
1117 logging.enable_python_logging = True
1118 # modify logging to remove the useless module: lines
1119 for level in LogLevel.values.values():
1120 logging.set_info(level, LogInfo.LEVEL | LogInfo.MESSAGE)
1121
1122 # Ok, some people prefer `-` in the executable name for tab completion so lets
1123 # support that by just splitting the executable name
1124 sys.argv[0:1] = os.path.basename(sys.argv[0]).split('-')
1125
1126 # parse argument definition for all sub commands
1127 parser = get_argument_parser()
1128
1129 # done, all functions parsed. Create the database instance and call the
1130 # correct subfunction according to the selected argument
1131 args = parser.parse_args()
1132
1133 if args.debugging:
1134 enable_debugging()
1135
1136 # manage some common options for up and downloading. slightly hacky but
1137 # need to be given to ConditionsDB on construction so meh
1138 nprocess = getattr(args, "nprocess", 1)
1139 retries = getattr(args, "retries", 0)
1140 # need at least one worker thread
1141 if nprocess <= 0:
1142 B2WARNING("-j must be larger than zero, ignoring")
1143 args.nprocess = nprocess = 1
1144
1145 conditions_db = ConditionsDB(args.base_url, nprocess, retries)
1146
1147 try:
1148 return args.func(args, conditions_db)
1149 except ConditionsDB.RequestError as e:
1150 B2ERROR(str(e))
1151 return 1
__call__(self, parser, namespace, values, option_string=None)
Definition cli_main.py:994
print_subparsers(self, parser, prefix="")
Definition cli_main.py:977
Definition main.py:1