Belle II Software light-2406-ragdoll
cli_main.py
1#!/usr/bin/env python3
2
3
10
11"""
12This tool provides a command line interface to all the tasks related to the
13:ref:`Conditions database <conditionsdb_overview>`: manage globaltags and iovs
14as well as upload new payloads or download of existing payloads.
15
16The usage of this tool is similar to git: there are sub commands like for
17example ``tag`` which groups all actions related to the management of
18globaltags. All the available commands are listed below.
19
20While the read access to the conditions database is always allowed (e.g.
21for downloading globaltags and payloads), users need a valid JSON Web Token
22(JWT) to authenticate to the conditions database when using this tool for
23creating/manpipulating globaltags or uploading payloads. For practical purposes,
24it is only necessary to know that a JWT is a string containing crypted
25information, and that string is stored in a file. More informations about what
26a JWT is can be found on
27`Wikipedia <https://en.wikipedia.org/wiki/JSON_Web_Token>`_.
28
29.. warning::
30
31 By default, users can only create globaltags whose name starts with
32 ``user_<username>_`` or ``temp_<username>_``, where ``username`` is the
33 B2MMS username.
34
35The tool automatically queries the JWT issuing server
36(https://token.belle2.org) and gets a valid token by asking the B2MMS username
37and password. The issued "default" JWT has a validity of 1 hour; after it
38expires, a new JWT needs to be obtained for authenticating the conditions
39database. When retrieved by this tool, the JWT is stored locally in the file
40``${HOME}/b2cdb_${BELLE2_USER}.token``.
41
42Some selected users (e.g. the calibration managers) are granted a JWT with an
43extended validity (30 days) to allow smooth operations with some automated
44workflows. Such "extended" JWTs are issued by a different server
45(https://token.belle2.org/extended). B2MMS username and password are necessary
46for getting the extended JWT. The extended JWT differs from the default JWT
47only by its validity and can be obtained only by manually querying the
48alternative server. If queried via web browser, a file containing the extended
49JWT will be downloaded in case the user is granted it. The server can also be
50queried via command line using
51``wget --user USERNAME --ask-password --no-check-certificate https://token.belle2.org/extended``
52or ``curl -u USERNAME -k https://token.belle2.org/extended``.
53
54If the environment variable ``${BELLE2_CDB_AUTH_TOKEN}`` is defined and points
55to a file containing a valid JWT, the ``b2conditionsdb`` tools use this token
56to authenticate with the CDB instead of querying the issuing server. This is
57useful when using an extended token. Please note that, in case the JWT to which
58``${BELLE2_CDB_AUTH_TOKEN}`` points is not valid anymore, the
59``b2conditionsdb`` tools will attempt to get a new one and store it into
60``${BELLE2_CDB_AUTH_TOKEN}``. If a new extended token is needed, it has to be
61manually obtained via https://token.belle2.org/extended and stored into
62``${BELLE2_CDB_AUTH_TOKEN}``.
63"""
64
65# Creation of command line parser is done semi-automatically: it looks for
66# functions which are called command_* and will take the * as sub command name
67# and the docstring as documentation: the first line as brief explanation and
68# the remainder as full documentation. command_foo_bar will create subcommand
69# bar inside command foo. The function will be called first with an instance of
70# argument parser where the command line options required for that command
71# should be added. If the command is run it will be called with the parsed
72# arguments first and an instance to the database object as second argument.
73
74# Some of the commands are separated out in separate modules named cli_*.py
75
76import os
77import re
78import sys
79import argparse
80import textwrap
81import json
82import difflib
83import shutil
84import pprint
85import requests
86from basf2 import B2ERROR, B2WARNING, B2INFO, LogLevel, LogInfo, logging, \
87 LogPythonInterface
88from basf2.utils import pretty_print_table
89from terminal_utils import Pager
90from dateutil.parser import parse as parse_date
91from getpass import getuser
92from conditions_db import ConditionsDB, enable_debugging, encode_name, PayloadInformation, set_cdb_authentication_token
93from conditions_db.cli_utils import ItemFilter
94from conditions_db.iov import IntervalOfValidity
95# the command_* functions are imported but not used so disable warning about
96# this if pylama/pylint is used to check
97from conditions_db.cli_upload import command_upload # noqa
98from conditions_db.cli_download import command_download, command_legacydownload # noqa
99from conditions_db.cli_management import command_tag_merge, command_tag_runningupdate, command_iovs, command_iovs_delete, command_iovs_copy, command_iovs_modify # noqa
100
101
102def escape_ctrl_chars(name):
103 """Remove ANSI control characters from strings"""
104 # compile the regex on first use and remember it
105 if not hasattr(escape_ctrl_chars, "_regex"):
106 escape_ctrl_chars._regex = re.compile("[\x00-\x1f\x7f-\x9f]")
107
108 # escape the control characters by putting them in as \xFF
109 def escape(match):
110 if match.group(0).isspace():
111 return match.group(0)
112 return f"\\x{ord(match.group(0)):02x}"
113
114 return escape_ctrl_chars._regex.sub(escape, name)
115
116
117def command_tag(args, db=None):
118 """
119 List, show, create, modify or clone globaltags.
120
121 This command allows to list, show, create modify or clone globaltags in the
122 central database. If no other command is given it will list all tags as if
123 ``%(prog)s show`` was given.
124 """
125
126 # no arguments to define, just a group command
127 if db is not None:
128 # normal call, in this case we just divert to list all tags
129 command_tag_list(args, db)
130
131
132def command_tag_list(args, db=None):
133 """
134 List all available globaltags.
135
136 This command allows to list all globaltags, optionally limiting the output
137 to ones matching a given search term. By default invalidated globaltags
138 will not be included in the list, to show them as well please add
139 ``--with-invalid`` as option. Alternatively one can use ``--only-published``
140 to show only tags which have been published
141
142 If the ``--regex`` option is supplied the search term will be interpreted
143 as a Python regular expression where the case is ignored.
144 """
145
146 tagfilter = ItemFilter(args)
147 if db is None:
148 # db object is none means we should just define arguments
149 args.add_argument("--detail", action="store_true", default=False,
150 help="show detailed information for all tags instead "
151 "of summary table")
152 group = args.add_mutually_exclusive_group()
153 group.add_argument("--with-invalid", action="store_true", default=False,
154 help="if given also invalidated tags will be shown")
155 group.add_argument("--only-published", action="store_true", default=False,
156 help="if given only published tags will be shown")
157 tagfilter.add_arguments("tags")
158 return
159
160 # see if the -f/-e/-r flags are ok
161 if not tagfilter.check_arguments():
162 return 1
163
164 req = db.request("GET", "/globalTags", f"Getting list of globaltags{tagfilter}")
165
166 # now let's filter the tags
167 taglist = []
168 for item in req.json():
169 if not tagfilter.check(item["name"]):
170 continue
171 tagstatus = item["globalTagStatus"]["name"]
172 if not getattr(args, "with_invalid", False) and tagstatus == "INVALID":
173 continue
174 if getattr(args, "only_published", False) and tagstatus != "PUBLISHED":
175 continue
176 taglist.append(item)
177
178 # done, sort by name
179 taglist.sort(key=lambda x: x["name"])
180
181 # and print, either detailed info for each tag or summary table at the end
182 table = []
183 with Pager(f"List of globaltags{tagfilter}{' (detailed)' if getattr(args, 'detail', False) else ''}", True):
184 for item in taglist:
185 if getattr(args, "detail", False):
186 print_globaltag(db, item)
187 else:
188 table.append([
189 item["globalTagId"],
190 item["name"],
191 escape_ctrl_chars(item.get("description", "")),
192 item["globalTagType"]["name"],
193 item["globalTagStatus"]["name"],
194 item["payloadCount"],
195 ])
196
197 if not getattr(args, "detail", False):
198 table.insert(0, ["id", "name", "description", "type", "status", "# payloads"])
199 pretty_print_table(table, [-10, 0, "*", -10, -10, -10], min_flexible_width=32)
200
201
202def print_globaltag(db, *tags):
203 """ Print detailed globaltag information for the given globaltags side by side"""
204 results = [["id"], ["name"], ["description"], ["type"], ["status"],
205 ["# payloads"], ["created"], ["modified"], ["modified by"]]
206 for info in tags:
207 if info is None:
208 continue
209
210 if isinstance(info, str):
211 try:
212 req = db.request("GET", f"/globalTag/{encode_name(info)}",
213 f"Getting info for globaltag {info}")
214 except ConditionsDB.RequestError as e:
215 # ok, there's an error for this one, let's continue with the other
216 # ones
217 B2ERROR(str(e))
218 continue
219
220 info = req.json()
221
222 created = parse_date(info["dtmIns"])
223 modified = parse_date(info["dtmMod"])
224 results[0].append(str(info["globalTagId"])),
225 results[1].append(info["name"]),
226 results[2].append(escape_ctrl_chars(info.get("description", "")))
227 results[3].append(info["globalTagType"]["name"])
228 results[4].append(info["globalTagStatus"]["name"]),
229 results[5].append(info["payloadCount"]),
230 results[6].append(created.astimezone(tz=None).strftime("%Y-%m-%d %H:%M:%S local time"))
231 results[7].append(modified.astimezone(tz=None).strftime("%Y-%m-%d %H:%M:%S local time"))
232 results[8].append(escape_ctrl_chars(info["modifiedBy"]))
233
234 ntags = len(results[0]) - 1
235 if ntags > 0:
236 pretty_print_table(results, [11] + ['*'] * ntags, True)
237 return ntags
238
239
240def change_state(db, tag, state, force=False):
241 """Change the state of a global tag
242
243 If the new state is not revertible then ask for confirmation
244 """
245 state = state.upper()
246 if state in ["INVALID", "PUBLISHED"] and not force:
247 name = input(f"ATTENTION: Marking a tag as {state} cannot be undone.\n"
248 "If you are sure you want to continue it please enter the tag name again: ")
249 if name != tag:
250 B2ERROR("Names don't match, aborting")
251 return 1
252
253 db.request("PUT", f"/globalTag/{encode_name(tag)}/updateGlobalTagStatus/{state}",
254 f"Changing globaltag state {tag} to {state}")
255
256
257def command_tag_show(args, db=None):
258 """
259 Show details about globaltags
260
261 This command will show details for the given globaltags like name,
262 description and number of payloads.
263 """
264
265 # this one is a bit similar to command_tag_list but gets single tag
266 # information from the database and thus no need for filtering. It will
267 # always show the detailed information
268
269 if db is None:
270 args.add_argument("tag", metavar="TAGNAME", nargs="+", help="globaltags to show")
271 return
272
273 # we retrieved all we could, print them
274 ntags = 0
275 with Pager("globaltag Information", True):
276 for tag in args.tag:
277 ntags += print_globaltag(db, tag)
278
279 # return the number of tags which could not get retrieved
280 return len(args.tag) - ntags
281
282
283def command_tag_create(args, db=None):
284 """
285 Create a new globaltag
286
287 This command creates a new globaltag in the database with the given name
288 and description. The name can only contain alpha-numeric characters and the
289 characters ``+-_:``.
290
291 .. warning::
292
293 By default, users can only create globaltags whose name starts with
294 ``user_<username>_`` or ``temp_<username>_``, where ``username`` is the
295 B2MMS username.
296 """
297
298 if db is None:
299 args.add_argument("type", metavar="TYPE", help="type of the globaltag to create, usually one of DEV or RELEASE")
300 args.add_argument("tag", metavar="TAGNAME", help="name of the globaltag to create")
301 args.add_argument("description", metavar="DESCRIPTION", help="description of the globaltag")
302 args.add_argument("-u", "--user", metavar="USER", help="username who created the tag. "
303 "If not given we will try to supply a useful default")
304 return
305
306 set_cdb_authentication_token(db, args.auth_token)
307
308 # create tag info needed for creation
309 info = {"name": args.tag, "description": args.description, "modifiedBy": args.user, "isDefault": False}
310 # add user information if not given by command line
311 if args.user is None:
312 info["modifiedBy"] = os.environ.get("BELLE2_USER", getuser())
313
314 typeinfo = db.get_globalTagType(args.type)
315 if typeinfo is None:
316 return 1
317
318 req = db.request("POST", f"/globalTag/{encode_name(typeinfo['name'])}",
319 "Creating globaltag {name}".format(**info),
320 json=info)
321 B2INFO("Successfully created globaltag {name} (id={globalTagId})".format(**req.json()))
322
323
324def command_tag_modify(args, db=None):
325 """
326 Modify a globaltag by changing name or description
327
328 This command allows to change the name or description of an existing globaltag.
329 You can supply any combination of -n,-d,-t and only the given values will be changed
330 """
331 if db is None:
332 args.add_argument("tag", metavar="TAGNAME", help="globaltag to modify")
333 args.add_argument("-n", "--name", help="new name")
334 args.add_argument("-d", "--description", help="new description")
335 args.add_argument("-t", "--type", help="new type of the globaltag")
336 args.add_argument("-u", "--user", metavar="USER", help="username who created the tag. "
337 "If not given we will try to supply a useful default")
338 args.add_argument("-s", "--state", help="new globaltag state, see the command ``tag state`` for details")
339 return
340
341 set_cdb_authentication_token(db, args.auth_token)
342
343 # first we need to get the old tag information
344 req = db.request("GET", f"/globalTag/{encode_name(args.tag)}",
345 f"Getting info for globaltag {args.tag}")
346
347 # now we update the tag information
348 info = req.json()
349 old_name = info["name"]
350 changed = False
351 for key in ["name", "description"]:
352 value = getattr(args, key)
353 if value is not None and value != info[key]:
354 info[key] = value
355 changed = True
356
357 info["modifiedBy"] = os.environ.get("BELLE2_USER", os.getlogin()) if args.user is None else args.user
358
359 if args.type is not None:
360 # for the type we need to know which types are defined
361 typeinfo = db.get_globalTagType(args.type)
362 if typeinfo is None:
363 return 1
364 # seems so, ok modify the tag info
365 if info['globalTagType'] != typeinfo:
366 info["globalTagType"] = typeinfo
367 changed = True
368
369 # and push the changed info to the server
370 if changed:
371 db.request("PUT", "/globalTag",
372 "Modifying globaltag {} (id={globalTagId})".format(old_name, **info),
373 json=info)
374
375 if args.state is not None:
376 name = args.name if args.name is not None else old_name
377 return change_state(db, name, args.state)
378
379
380def command_tag_clone(args, db=None):
381 """
382 Clone a given globaltag including all IoVs
383
384 This command allows to clone a given globaltag with a new name but still
385 containing all the IoVs defined in the original globaltag.
386 """
387
388 if db is None:
389 args.add_argument("tag", metavar="TAGNAME", help="globaltag to be cloned")
390 args.add_argument("name", metavar="NEWNAME", help="name of the cloned globaltag")
391 return
392
393 set_cdb_authentication_token(db, args.auth_token)
394
395 # first we need to get the old tag information
396 req = db.request("GET", f"/globalTag/{encode_name(args.tag)}",
397 f"Getting info for globaltag {args.tag}")
398 info = req.json()
399
400 # now we clone the tag. id came from the database so no need for escape
401 req = db.request("POST", "/globalTags/{globalTagId}".format(**info),
402 "Cloning globaltag {name} (id={globalTagId})".format(**info))
403
404 # it gets a stupid name "{ID}_copy_of_{name}" so we change it to something
405 # nice like the user asked
406 cloned_info = req.json()
407 cloned_info["name"] = args.name
408 # and push the changed info to the server
409 db.request("PUT", "/globalTag", "Renaming globaltag {name} (id={globalTagId})".format(**cloned_info),
410 json=cloned_info)
411
412
413def command_tag_state(args, db):
414 """
415 Change the state of a globaltag.
416
417 This command changes the state of a globaltag to the given value.
418
419 Usually the valid states are
420
421 OPEN
422 Tag can be modified, payloads and iovs can be created and deleted. This
423 is the default state for new or cloned globaltags and is not suitable
424 for use in data analysis
425
426 Can be transitioned to TESTING, RUNNING
427
428 TESTING
429 Tag cannot be modified and is suitable for testing but can be reopened
430
431 Can be transitioned to VALIDATED, OPEN
432
433 VALIDATED
434 Tag cannot be modified and has been tested.
435
436 Can be transitioned to PUBLISHED, OPEN
437
438 PUBLISHED
439 Tag cannot be modified and is suitable for user analysis
440
441 Can only be transitioned to INVALID
442
443 RUNNING
444 Tag can only be modified by adding new runs, not modifying the payloads
445 for existing runs.
446
447 INVALID:
448 Tag is invalid and should not be used for anything.
449
450 This state is end of life for a globaltag and cannot be transitioned to
451 any other state.
452
453 .. versionadded:: release-04-00-00
454 """
455 if db is None:
456 args.add_argument("tag", metavar="TAGNAME", help="globaltag to be changed")
457 args.add_argument("state", metavar="STATE", help="new state for the globaltag")
458 args.add_argument("--force", default=False, action="store_true", help=argparse.SUPPRESS)
459 return
460
461 set_cdb_authentication_token(db, args.auth_token)
462
463 return change_state(db, args.tag, args.state, args.force)
464
465
466def remove_repeated_values(table, columns, keep=None):
467 """Strip repeated values from a table of values
468
469 This function takes a table (a list of lists with all the same length) and
470 removes values in certain columns if they are identical in consecutive rows.
471
472 It does this in a dependent way only if the previous columns are identical
473 will it continue stripping further columns. For example, given the table ::
474
475 table = [
476 ["A", "a"],
477 ["B", "a"],
478 ["B", "a"],
479 ["B", "b"],
480 ]
481
482 If we want to remove duplicates in all columns in order it would look like this:
483
484 >>> remove_repeated_values(table, [0,1])
485 [
486 ["A", "a"],
487 ["B", "a"],
488 [ "", ""],
489 [ "", "b"],
490 ]
491
492 But we can give selected columns to strip one after the other
493
494 >>> remove_repeated_values(table, [1,0])
495 [
496 ["A", "a"],
497 ["B", ""],
498 [ "", ""],
499 ["B", "b"],
500 ]
501
502 In addition, we might want to only strip some columns if previous columns
503 were identical but keep the values of the previous column. For this one can
504 supply ``keep``:
505
506 >>> remove_repeated_values(table, [0,1,2], keep=[0])
507 [
508 ["A", "a"],
509 ["B", "a"],
510 ["B", ""],
511 ["B", "b"],
512 ]
513
514 Parameters:
515 table (list(list(str))): 2d table of values
516 columns (list(int)): indices of columns to consider in order
517 keep (set(int)): indices of columns to not strip
518 """
519 last_values = [None] * len(columns)
520 for row in table[1:]:
521 current_values = [row[i] for i in columns]
522 for i, curr, last in zip(columns, current_values, last_values):
523 if curr != last:
524 break
525
526 if keep and i in keep:
527 continue
528
529 row[i] = ""
530
531 last_values = current_values
532
533
534def command_diff(args, db):
535 """
536 Compare two globaltags
537
538 This command allows to compare two globaltags. It will show the changes in
539 a format similar to a unified diff but by default it will not show any
540 context, only the new or removed payloads. Added payloads are marked with a
541 ``+`` in the first column, removed payloads with a ``-``
542
543 If ``--full`` is given it will show all payloads, even the ones common to
544 both globaltags. The differences can be limited to a given run and
545 limited to a set of payloads names using ``--filter`` or ``--exclude``. If
546 the ``--regex`` option is supplied the search term will be interpreted as a
547 python regular expression where the case is ignored.
548
549 .. versionchanged:: release-03-00-00
550 modified output structure and added ``--human-readable``
551 .. versionchanged:: after release-04-00-00
552 added parameter ``--checksums`` and ``--show-ids``
553 """
554 iovfilter = ItemFilter(args)
555 if db is None:
556 args.add_argument("--full", default=False, action="store_true",
557 help="If given print all iovs, also those which are the same in both tags")
558 args.add_argument("--run", type=int, nargs=2, metavar="N", help="exp and run numbers "
559 "to limit showing iovs to a ones present in a given run")
560 args.add_argument("--human-readable", default=False, action="store_true",
561 help="If given the iovs will be written in a more human friendly format. "
562 "Also repeated payload names will be omitted to create a more readable listing.")
563 args.add_argument("--checksums", default=False, action="store_true",
564 help="If given don't show the revision number but the md5 checksum")
565 args.add_argument("--show-ids", default=False, action="store_true",
566 help="If given also show the payload and iov ids for each iov")
567
568 args.add_argument("tagA", metavar="TAGNAME1", help="base for comparison")
569 args.add_argument("tagB", metavar="TAGNAME2", help="tagname to compare")
570 args.add_argument("--run-range", nargs=4, default=None, type=int,
571 metavar=("FIRST_EXP", "FIRST_RUN", "FINAL_EXP", "FINAL_RUN"),
572 help="Can be four numbers to limit the run range to be compared"
573 "Only iovs overlapping, even partially, with this range will be considered.")
574 iovfilter.add_arguments("payloads")
575 return
576
577 # check arguments
578 if not iovfilter.check_arguments():
579 return 1
580
581 with Pager(f"Differences between globaltags {args.tagA} and {args.tagB}{iovfilter}", True):
582 print("globaltags to be compared:")
583 ntags = print_globaltag(db, args.tagA, args.tagB)
584 if ntags != 2:
585 return 1
586 print()
587 listA = [
588 e for e in db.get_all_iovs(
589 args.tagA,
590 message=str(iovfilter),
591 run_range=args.run_range) if iovfilter.check(
592 e.name)]
593 listB = [
594 e for e in db.get_all_iovs(
595 args.tagB,
596 message=str(iovfilter),
597 run_range=args.run_range) if iovfilter.check(
598 e.name)]
599
600 B2INFO("Comparing contents ...")
601 diff = difflib.SequenceMatcher(a=listA, b=listB)
602 table = [["", "Name", "Rev" if not args.checksums else "Checksum"]]
603 columns = [1, "+", -8 if not args.checksums else -32]
604
605 if args.human_readable:
606 table[0] += ["Iov"]
607 columns += [-36]
608 else:
609 table[0] += ["First Exp", "First Run", "Final Exp", "Final Run"]
610 columns += [6, 6, 6, 6]
611
612 if args.show_ids:
613 table[0] += ["IovId", "PayloadId"]
614 columns += [7, 9]
615
616 def add_payloads(opcode, payloads):
617 """Add a list of payloads to the table, filling the first column with opcode"""
618 for p in payloads:
619 row = [opcode, p.name, p.revision if not args.checksums else p.checksum]
620 if args.human_readable:
621 row += [p.readable_iov()]
622 else:
623 row += list(p.iov)
624
625 if args.show_ids:
626 row += [p.iov_id, p.payload_id]
627 table.append(row)
628
629 for tag, i1, i2, j1, j2 in diff.get_opcodes():
630 if tag == "equal":
631 if not args.full:
632 continue
633 add_payloads(" ", listB[j1:j2])
634 if tag in ["delete", "replace"]:
635 add_payloads("-", listA[i1:i2])
636 if tag in ["insert", "replace"]:
637 add_payloads("+", listB[j1:j2])
638
639 if args.human_readable:
640 # strip repeated names, revision, payloadid, to make it more readable.
641 # this is dependent on the fact that the opcode is still the same but we
642 # don't want to strip the opcode ...
643 remove_repeated_values(table, [0, 1, 2] + ([-1] if args.show_ids else []), keep=[0])
644
645 def color_row(row, widths, line):
646 if not LogPythonInterface.terminal_supports_colors():
647 return line
648 begin = {'+': '\x1b[32m', '-': '\x1b[31m'}.get(row[0], "")
649 end = '\x1b[0m'
650 return begin + line + end
651
652 # print the table but make sure the first column is empty except for
653 # added/removed lines so that it can be copy-pasted into a diff syntax
654 # highlighting area (say merge request description)
655 print(f" Differences between {args.tagA} and {args.tagB}")
656 pretty_print_table(table, columns, transform=color_row,
657 hline_formatter=lambda w: " " + (w - 1) * '-')
658
659
660def command_iov(args, db):
661 """
662 List all IoVs defined in a globaltag, optionally limited to a run range
663
664 This command lists all IoVs defined in a given globaltag. The list can be
665 limited to a given run and optionally searched using ``--filter`` or
666 ``--exclude``. If the ``--regex`` option is supplied the search term will
667 be interpreted as a Python regular expression where the case is ignored.
668
669 .. versionchanged:: release-03-00-00
670 modified output structure and added ``--human-readable``
671 .. versionchanged:: after release-04-00-00
672 added parameter ``--checksums`` and ``--show-ids``
673 .. versionchanged:: after release-08-00-04
674 added parameter ``--run-range``
675 """
676
677 iovfilter = ItemFilter(args)
678
679 if db is None:
680 args.add_argument("tag", metavar="TAGNAME", help="globaltag for which the the IoVs should be listed")
681 args.add_argument("--run", type=int, nargs=2, metavar="N", help="exp and run numbers "
682 "to limit showing iovs to a ones present in a given run")
683 args.add_argument("--detail", action="store_true", default=False,
684 help="if given show a detailed information for all "
685 "IoVs including details of the payloads")
686 args.add_argument("--human-readable", default=False, action="store_true",
687 help="If given the iovs will be written in a more human friendly format. "
688 "Also repeated payload names will be omitted to create a more readable listing.")
689 args.add_argument("--checksums", default=False, action="store_true",
690 help="If given don't show the revision number but the md5 checksum")
691 args.add_argument("--show-ids", default=False, action="store_true",
692 help="If given also show the payload and iov ids for each iov")
693 args.add_argument("--run-range", nargs=4, default=None, type=int,
694 metavar=("FIRST_EXP", "FIRST_RUN", "FINAL_EXP", "FINAL_RUN"),
695 help="Can be four numbers to limit the run range to be shown"
696 "Only iovs overlapping, even partially, with this range will be shown.")
697 iovfilter.add_arguments("payloads")
698 return
699
700 # check arguments
701 if not iovfilter.check_arguments():
702 return 1
703
704 # Check if the globaltag exists otherwise I get the same result for an emply global tag or for a non-existing one
705 if db.get_globalTagInfo(args.tag) is None:
706 B2ERROR(f"Globaltag '{args.tag}' doesn't exist.")
707 return False
708
709 run_range_str = f' valid in {tuple(args.run_range)}' if args.run_range else ''
710 args.run_range = IntervalOfValidity(args.run_range) if args.run_range else None
711
712 if args.run is not None:
713 msg = f"Obtaining list of iovs for globaltag {args.tag}, exp={args.run[0]}, run={args.run[1]}{iovfilter}"
714 req = db.request("GET", "/iovPayloads", msg, params={'gtName': args.tag, 'expNumber': args.run[0],
715 'runNumber': args.run[1]})
716 else:
717 msg = f"Obtaining list of iovs for globaltag {args.tag}{iovfilter}{run_range_str}"
718 req = db.request("GET", f"/globalTag/{encode_name(args.tag)}/globalTagPayloads", msg)
719
720 with Pager(f"List of IoVs{iovfilter}{run_range_str}{' (detailed)' if args.detail else ''}", True):
721 payloads = []
722 for item in req.json():
723 payload = item["payload" if 'payload' in item else "payloadId"]
724 if "payloadIov" in item:
725 iovs = [item['payloadIov']]
726 else:
727 iovs = item['payloadIovs']
728
729 if not iovfilter.check(payload['basf2Module']['name']):
730 continue
731
732 for iov in iovs:
733 if args.run_range is not None:
734 if IntervalOfValidity(
735 iov['expStart'], iov['runStart'], iov['expEnd'], iov['runEnd']
736 ).intersect(args.run_range) is None:
737 continue
738
739 if args.detail:
740 # detailed mode, show a table with all information for each
741 # iov
742 iov_created = parse_date(iov["dtmIns"])
743 iov_modified = parse_date(iov["dtmMod"])
744 payload_created = parse_date(payload["dtmIns"])
745 payload_modified = parse_date(payload["dtmMod"])
746 result = [
747 ["IoV Id", str(iov["payloadIovId"])],
748 ["first experiment", iov["expStart"]],
749 ["first run", iov["runStart"]],
750 ["final experiment", iov["expEnd"]],
751 ["final run", iov["runEnd"]],
752 ["IoV created", iov_created.astimezone(tz=None).strftime("%Y-%m-%d %H:%M:%S local time")],
753 ["IoV modified", iov_modified.astimezone(tz=None).strftime("%Y-%m-%d %H:%M:%S local time")],
754 ["IoV modified by", iov["modifiedBy"]],
755 ["payload Id", str(payload["payloadId"])],
756 ["name", payload["basf2Module"]["name"]],
757 ["revision", payload["revision"]],
758 ["checksum", payload["checksum"]],
759 ["payloadUrl", payload["payloadUrl"]],
760 ["baseUrl", payload.get("baseUrl", "")],
761 # print created and modified timestamps in local time zone
762 ["payload created", payload_created.astimezone(tz=None).strftime("%Y-%m-%d %H:%M:%S local time")],
763 ["payload modified", payload_modified.astimezone(tz=None).strftime("%Y-%m-%d %H:%M:%S local time")],
764 ["payload modified by", escape_ctrl_chars(payload["modifiedBy"])],
765 ]
766 print()
767 pretty_print_table(result, [-40, '*'], True)
768 else:
769 payloads.append(PayloadInformation.from_json(payload, iov))
770
771 if not args.detail:
772 def add_ids(table, columns, payloads):
773 """Add the numerical ids to the table"""
774 if args.show_ids:
775 table[0] += ["IovId", "PayloadId"]
776 columns += [9, 9]
777 for row, p in zip(table[1:], payloads):
778 row += [p.iov_id, p.payload_id]
779 payloads.sort()
780 if args.human_readable:
781 table = [["Name", "Rev" if not args.checksums else "Checksum", "IoV"]]
782 columns = ["+", -8 if not args.checksums else -32, -32]
783 table += [[p.name, p.revision if not args.checksums else p.checksum, p.readable_iov()] for p in payloads]
784 add_ids(table, columns, payloads)
785 # strip repeated names, revision, payloadid, to make it more readable
786 remove_repeated_values(table, columns=[0, 1] + ([-1] if args.show_ids else []))
787
788 else:
789 table = [["Name", "Rev" if not args.checksums else "Checksum", "First Exp", "First Run", "Final Exp", "Final Run"]]
790 table += [[p.name, p.revision if not args.checksums else p.checksum] + list(p.iov) for p in payloads]
791 columns = ["+", -8 if not args.checksums else -32, 6, 6, 6, 6]
792 add_ids(table, columns, payloads)
793
794 pretty_print_table(table, columns)
795
796
797def command_dump(args, db):
798 """
799 Dump the content of a given payload
800
801 .. versionadded:: release-03-00-00
802
803 This command will dump the payload contents stored in a given payload. One
804 can either specify the ``payloadId`` (from a previous output of
805 ``b2conditionsdb iov``), the payload name and its revision in the central
806 database, or directly specify a local database payload file.
807
808 .. rubric:: Examples
809
810 Dump the content of a previously downloaded payload file::
811
812 $ b2conditionsdb dump -f centraldb/dbstore_BeamParameters_rev_59449.root
813
814 Dump the content of a payload by name and revision directly from the central database::
815
816 $ b2conditionsdb dump -r BeamParameters 59449
817
818 Dump the content of the payload by name which is valid in a given globaltag
819 for a given experiment and run::
820
821 $ b2conditionsdb dump -g BeamParameters main_2021-08-04 0 0
822
823 Or directly by payload id from a previous call to ``b2conditionsdb iov``::
824
825 $ b2conditionsdb dump -i 59685
826
827 .. rubric:: Usage
828
829 Depending on whether you want to display a payload by its id in the
830 database, its name and revision in the database or from a local file
831 provide **one** of the arguments ``-i``, ``-r``, ``-f`` or ``-g``
832
833 .. versionchanged:: after release-04-00-00
834 added argument ``-r`` to directly dump a payload valid for a given run
835 in a given globaltag
836 """
837 if db is None:
838 group = args.add_mutually_exclusive_group(required=True)
839 choice = group.add_mutually_exclusive_group()
840 choice.add_argument("-i", "--id", metavar="PAYLOADID", help="payload id to dump")
841 choice.add_argument("-r", "--revision", metavar=("NAME", "REVISION"), nargs=2,
842 help="Name and revision of the payload to dump")
843 choice.add_argument("-f", "--file", metavar="FILENAME", help="Dump local payload file")
844 choice.add_argument("-g", "--valid", metavar=("NAME", "GLOBALTAG", "EXP", "RUN"), nargs=4,
845 help="Dump the payload valid for the given exp, run number in the given globaltag")
846 args.add_argument("--show-typenames", default=False, action="store_true",
847 help="If given show the type names of all classes. "
848 "This makes output more crowded but can be helpful for complex objects.")
849 args.add_argument("--show-streamerinfo", default=False, action="store_true",
850 help="If given show the StreamerInfo for the classes in the the payload file. "
851 "This can be helpful to find out which version of a payload object "
852 "is included and what are the members")
853
854 return
855
856 payload = None
857 # local file, don't query database at all
858 if args.file:
859 filename = args.file
860 if not os.path.isfile(filename):
861 B2ERROR(f"Payloadfile {filename} could not be found")
862 return 1
863
864 match = re.match(r"^dbstore_(.*)_rev_(.*).root$", os.path.basename(filename))
865 if not match:
866 match = re.match(r"^(.*)_r(.*).root$", os.path.basename(filename))
867 if not match:
868 B2ERROR("Filename doesn't follow database convention.\n"
869 "Should be 'dbstore_${payloadname}_rev_${revision}.root' or '${payloadname}_r${revision.root}'")
870 return 1
871 name = match.group(1)
872 revision = match.group(2)
873 payloadId = "Unknown"
874 else:
875 # otherwise do just that: query the database for either payload id or
876 # the name,revision
877 if args.id:
878 req = db.request("GET", f"/payload/{args.id}", "Getting payload info")
879 payload = PayloadInformation.from_json(req.json())
880 name = payload.name
881 elif args.revision:
882 name, rev = args.revision
883 rev = int(rev)
884 req = db.request("GET", f"/module/{encode_name(name)}/payloads", "Getting payload info")
885 for p in req.json():
886 if p["revision"] == rev:
887 payload = PayloadInformation.from_json(p)
888 break
889 else:
890 B2ERROR(f"Cannot find payload {name} with revision {rev}")
891 return 1
892 elif args.valid:
893 name, globaltag, exp, run = args.valid
894 payload = None
895 for p in db.get_all_iovs(globaltag, exp, run, f", name={name}"):
896 if p.name == name and (payload is None or p.revision > payload.revision):
897 payload = p
898
899 if payload is None:
900 B2ERROR(f"Cannot find payload {name} in globaltag {globaltag} for exp,run {exp},{run}")
901 return 1
902
903 filename = payload.url
904 revision = payload.revision
905 payloadId = payload.payload_id
906 del payload
907
908 # late import of ROOT because of all the side effects
909 from ROOT import TFile, TBufferJSON, cout
910
911 # remote http opening or local file
912 tfile = TFile.Open(filename)
913 json_str = None
914 raw_contents = None
915 if not tfile or not tfile.IsOpen():
916 # could be a non-root payload file
917 contents = db._session.get(filename, stream=True)
918 if contents.status_code != requests.codes.ok:
919 B2ERROR(f"Could not open payload file {filename}")
920 return 1
921 raw_contents = contents.raw.read().decode()
922 else:
923 obj = tfile.Get(name)
924 if obj:
925 json_str = TBufferJSON.ConvertToJSON(obj)
926
927 def drop_fbits(obj):
928 """
929 Drop some members from ROOT json output.
930
931 We do not care about fBits, fUniqueID or the typename of sub classes,
932 we assume users are only interested in the data stored in the member
933 variables
934 """
935 obj.pop("fBits", None)
936 obj.pop("fUniqueID", None)
937 if not args.show_typenames:
938 obj.pop("_typename", None)
939 return obj
940
941 with Pager(f"Contents of Payload {name}, revision {revision} (id {payloadId})", True):
942 if args.show_streamerinfo and tfile:
943 B2INFO("StreamerInfo of Payload {name}, revision {revision} (id {payloadId})")
944 tfile.ShowStreamerInfo()
945 # sadly this prints to std::cout or even stdout but doesn't flush ... so we have
946 # to make sure std::cout is flushed before printing anything else
947 cout.flush()
948 # and add a newline
949 print()
950
951 if json_str is not None:
952 B2INFO(f"Contents of Payload {name}, revision {revision} (id {payloadId})")
953 # load the json as python object dropping some things we don't want to
954 # print
955 obj = json.loads(json_str.Data(), object_hook=drop_fbits)
956 # print the object content using pretty print with a certain width
957 pprint.pprint(obj, compact=True, width=shutil.get_terminal_size((80, 20))[0])
958 elif raw_contents:
959 B2INFO(f"Raw contents of Payload {name}, revision {revision} (id {payloadId})")
960 print(escape_ctrl_chars(raw_contents))
961 elif tfile:
962 B2INFO(f"ROOT contents of Payload {name}, revision {revision} (id {payloadId})")
963 B2WARNING("The payload is a valid ROOT file but doesn't contain a payload object with the expected name. "
964 " Automated display of file contents are not possible, showing just entries in the ROOT file.")
965 tfile.ls()
966
967
968class FullHelpAction(argparse._HelpAction):
969 """Class to recursively show help for an ArgumentParser and all it's sub_parsers"""
970
971 def print_subparsers(self, parser, prefix=""):
972 """Print help message for given parser and call again for all sub parsers"""
973 # retrieve subparsers from parser
974 subparsers_actions = [
975 action for action in parser._actions
976 if isinstance(action, argparse._SubParsersAction)]
977 # there will probably only be one subparser_action,
978 # but better save than sorry
979 for subparsers_action in subparsers_actions:
980 # get all subparsers and print help
981 for choice, subparser in subparsers_action.choices.items():
982 print()
983 print(f"Command '{prefix}{choice}'")
984 print(subparser.format_help())
985
986 self.print_subparsers(subparser, prefix=f"{prefix}{choice} ")
987
988 def __call__(self, parser, namespace, values, option_string=None):
989 """Show full help message"""
990 # run in pager because amount of options will be looong
991 with Pager(f"{parser.prog} {option_string}"):
992 parser.print_help()
993 self.print_subparsers(parser)
994 parser.exit()
995
996
997def get_argument_parser():
998 """
999 Build a parser with all arguments of all commands
1000 """
1001 # extra ArgumentParser with the global options just for reusability
1002 options = argparse.ArgumentParser(add_help=False)
1003 options.add_argument("--debugging", action="store_true",
1004 help="Enable debugging of http traffic")
1005 options.add_argument("--help-full", action=FullHelpAction,
1006 help="show help message for all commands and exit")
1007 options.add_argument("--base-url", default=None,
1008 help="URI for the base of the REST API, if not given a list of default locations is tried")
1009 options.add_argument("--auth-token", type=str, default=None,
1010 help="JSON Web Token necessary for authenticating to the conditions database. "
1011 "Useful only for debugging, since by default the tool automatically "
1012 "gets a token for you by asking the B2MMS username and password. "
1013 "If the environment variable ``$BELLE2_CDB_AUTH_TOKEN`` points to a file with a valid "
1014 "token, such token is used (useful for automatic workflows).")
1015
1016 parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, parents=[options])
1017 parser.set_defaults(func=lambda x, y: parser.print_help())
1018 parsers = parser.add_subparsers(
1019 title="Top level commands",
1020 description="To get additional help, run '%(prog)s COMMAND --help'"
1021 )
1022
1023 subparsers = {}
1024 # now we go through all the functions defined which start with command_
1025 for name, func in sorted(globals().items()):
1026 if not name.startswith("command_"):
1027 continue
1028 # we interpret command_foo_bar_baz as subcommand baz of subcommand bar
1029 # of subcommand foo. So let's split this into all commands and remove
1030 # the command_
1031 parts = name.split('_')[1:]
1032 # now we need to get the subparsers instance for the parent command. if
1033 # the command is top level, e.g. foo, we just use parsers. Otherwise we
1034 # go look into the dict of subparsers for command chains.
1035 parent = parsers
1036 if(len(parts) > 1):
1037 parent_parser, parent = subparsers[tuple(parts[:-1])]
1038 # if we are the first subcommand to a given command we have to add
1039 # the subparsers. do that and add it back to the dict
1040 if parent is None:
1041 parent = parent_parser.add_subparsers(
1042 title="sub commands",
1043 description="To get additional help, run '%(prog)s COMMAND --help'"
1044 )
1045 subparsers[tuple(parts[:-1])][1] = parent
1046 # so we have our subparsers instance, now create argument parser for the
1047 # function. We use the first part of the function docstring as help text
1048 # and everything after the first empty line as description of the
1049 # command
1050 helptxt, description = textwrap.dedent(func.__doc__).split("\n\n", 1)
1051 command_parser = parent.add_parser(parts[-1], help=helptxt, add_help=True, description=description,
1052 parents=[options], formatter_class=argparse.RawDescriptionHelpFormatter)
1053 # now call the function with the parser as first argument and no
1054 # database instance. This let's them define their own arguments
1055 func(command_parser, None)
1056 # and set the function as default to be called for later
1057 command_parser.set_defaults(func=func)
1058 # also add it back to the list of subparsers
1059 subparsers[tuple(parts)] = [command_parser, None]
1060
1061 return parser
1062
1063
1064def create_symlinks(base):
1065 """Create symlinks from base to all subcommands.
1066
1067 e.g. if the base is ``b2conditionsdb`` then this command will create symlinks
1068 like ``b2conditionsdb-tag-show`` in the same directory
1069
1070 When adding a new command to b2conditionsdb this function needs to be executed
1071 in the framework tools directory
1072
1073 python3 -c 'from conditions_db import cli_main; cli_main.create_symlinks("b2conditionsdb")'
1074 """
1075 import os
1076 excluded = [
1077 ['tag'] # the tag command without subcommand is not very useful
1078 ]
1079 for name in sorted(globals().keys()):
1080 if not name.startswith("command_"):
1081 continue
1082 parts = name.split("_")[1:]
1083 if parts in excluded:
1084 continue
1085 dest = base + "-".join([""] + parts)
1086
1087 try:
1088 os.remove(dest)
1089 except FileNotFoundError:
1090 pass
1091 print(f"create symlink {dest}")
1092 os.symlink(base, dest)
1093
1094
1095def main():
1096 """
1097 Main function for the command line interface.
1098
1099 it will automatically create an ArgumentParser including all functions which
1100 start with command_ in the global namespace as sub commands. These
1101 functions should take the arguments as first argument and an instance of the
1102 ConditionsDB interface as second argument. If the db interface is None the
1103 first argument is an instance of argparse.ArgumentParser an in this case the
1104 function should just add all needed arguments to the argument parser and
1105 return.
1106 """
1107
1108 # disable error summary
1109 logging.enable_summary(False)
1110 # log via python stdout to be able to capture
1111 logging.enable_python_logging = True
1112 # modify logging to remove the useless module: lines
1113 for level in LogLevel.values.values():
1114 logging.set_info(level, LogInfo.LEVEL | LogInfo.MESSAGE)
1115
1116 # Ok, some people prefer `-` in the executable name for tab completion so lets
1117 # support that by just splitting the executable name
1118 sys.argv[0:1] = os.path.basename(sys.argv[0]).split('-')
1119
1120 # parse argument definition for all sub commands
1121 parser = get_argument_parser()
1122
1123 # done, all functions parsed. Create the database instance and call the
1124 # correct subfunction according to the selected argument
1125 args = parser.parse_args()
1126
1127 if args.debugging:
1128 enable_debugging()
1129
1130 # manage some common options for up and downloading. slightly hacky but
1131 # need to be given to ConditionsDB on construction so meh
1132 nprocess = getattr(args, "nprocess", 1)
1133 retries = getattr(args, "retries", 0)
1134 # need at least one worker thread
1135 if nprocess <= 0:
1136 B2WARNING("-j must be larger than zero, ignoring")
1137 args.nprocess = nprocess = 1
1138
1139 conditions_db = ConditionsDB(args.base_url, nprocess, retries)
1140
1141 try:
1142 return args.func(args, conditions_db)
1143 except ConditionsDB.RequestError as e:
1144 B2ERROR(str(e))
1145 return 1
def print_subparsers(self, parser, prefix="")
Definition: cli_main.py:971
def __call__(self, parser, namespace, values, option_string=None)
Definition: cli_main.py:988
Definition: main.py:1