Belle II Software light-2406-ragdoll
cli_management.py
1
8import functools
9import time
10from fnmatch import fnmatch
11from concurrent.futures import ThreadPoolExecutor
12from collections import defaultdict
13from basf2 import B2INFO, B2ERROR, B2WARNING, LogPythonInterface # noqa
14from basf2.utils import pretty_print_table
15from terminal_utils import Pager
16from conditions_db import set_cdb_authentication_token
17from conditions_db.iov import IoVSet, IntervalOfValidity
18from conditions_db.runningupdate import RunningTagUpdater, RunningTagUpdaterError, RunningTagUpdateMode
19from conditions_db.cli_utils import ItemFilter
20
21
22def get_all_iovsets(existing_payloads, run_range=None):
23 """Given a list of PayloadInformation objects, return a reduced list PayloadInformation
24 objects with the single iovs replaced with IoVSets. Payloads with the same
25 name and revision will be merged.
26
27 Overlaps will raise an B2ERROR
28 """
29 all_payloads = defaultdict(lambda: IoVSet(allow_overlaps=True))
30 by_name = defaultdict(lambda: IoVSet(allow_overlaps=False))
31 infos = {}
32 for payload in existing_payloads:
33 # we want to make set of iovs for each payload
34 iov = IntervalOfValidity(payload.iov)
35 # possibly we have a run range we want to limit to
36 if run_range is not None:
37 iov &= run_range
38 if not iov:
39 continue
40
41 # merge the iovs for the same revision
42 all_payloads[payload.name, payload.revision].add(iov)
43 # also keep the PayloadInformation
44 infos[payload.name, payload.revision] = payload
45
46 # and also check if there are any overlaps with the same payload name
47 try:
48 by_name[payload.name].add(iov)
49 except ValueError as e:
50 B2ERROR(f"Overlap for payload {payload.name} r{payload.revision}: {e}")
51
52 # so now flatten the thing again and return PayloadInformation objects we slightly modify
53 result = []
54 for (name, revision), iov in all_payloads.items():
55 info = infos[name, revision]
56 info.iov = iov
57 info.iov_id = None
58 result.append(info)
59
60 return result
61
62
63def create_iov_wrapper(db, globaltag_id, payload):
64 """
65 Wrapper function for adding payloads into a given globaltag.
66 """
67 for iov in payload.iov:
68 if db.create_iov(globaltag_id, payload.payload_id, *iov.tuple) is None:
69 raise RuntimeError(f"Cannot create iov for {payload.name} r{payload.revision}")
70
71
72def command_tag_merge(args, db=None):
73 """
74 Merge a list of globaltags in the order they are given.
75
76 This command allows to merge a number of globaltags into a single globaltag.
77 Payloads from later globaltags in the list of arguments are used to fill gaps
78 present in earlier globaltags.
79
80 The result is equivalent to having multiple globaltags setup in the conditions
81 access for basf2 (highest priority goes first).
82
83 .. warning::
84
85 The order of the globaltags is highest priority first, so payloads from
86 globaltags earlier on the command line will be taken with before globaltags
87 from later tags.
88
89 For each globaltag in the list we copy all payloads to the output globaltag
90 if there is no payload of that name valid for the given interval of validity
91 in any previous globaltags in the list.
92
93 If the payload overlaps partially with a payload from a previous globaltag
94 in the list the interval of validity is shortened (and possibly split) to
95 not overlap but to just fill the gaps.
96
97 For example:
98
99 Globaltag ``A`` contains ::
100
101 payload1, rev 2, valid from 1,0 to 1,10
102 payload1, rev 3, valid from 1,20 to 1,22
103 payload2, rev 1, valid from 1,0 to 1,-1
104
105 Globaltag ``B`` contains ::
106
107 payload1, rev 1, valid from 1,1 to 1,30
108 payload2, rev 2, valid from 0,1 to 1,20
109
110 Then running ``b2conditionsdb tag merge -o C A B``, the output globaltag ``C``
111 after the merge will contain::
112
113 payload1, rev 2, valid from 1,0 to 1,10
114 payload1, rev 1, valid from 1,11 to 1,19
115 payload1, rev 3, valid from 1,20 to 1,22
116 payload1, rev 1, valid from 1,23 to 1,30
117 payload2, rev 2, valid from 0,1 to 0,-1
118 payload2, rev 1, valid from 1,0 to 1,-1
119
120 When finished, this command will print a table of payloads and their
121 validity and from which globaltag they were taken. If ``--dry-run`` is given
122 it will only print the list of payloads.
123
124 Optionally one can specify ``--run-range`` to limit the run range for which
125 the merging should take place. In the example above, running with
126 ``--run-range 1 0 1 21`` the result would be ::
127
128 payload1, rev 2, valid from 1,0 to 1,10
129 payload1, rev 1, valid from 1,11 to 1,19
130 payload1, rev 3, valid from 1,20 to 1,21
131 payload2, rev 1, valid from 1,0 to 1,21
132
133 .. versionadded:: release-05-01-00
134 """
135
136 if db is None:
137 args.add_argument("globaltag", nargs="+", help="name of the globaltag")
138 group = args.add_argument_group("required named arguments")
139 group.add_argument("-o", "--output", required=True, help="Name of the output globaltag")
140 args.add_argument("--dry-run", help="Don't do anything, just print a table with the results",
141 action="store_true", default=False)
142 args.add_argument("--run-range", nargs=4, default=None, type=int,
143 metavar=("FIRST_EXP", "FIRST_RUN", "FINAL_EXP", "FINAL_RUN"),
144 help="Can be for numbers to limit the run range to put"
145 "in the output globaltag: All iovs will be limited to "
146 "be in this range.")
147 args.add_argument("-j", type=int, default=10, dest="nprocess",
148 help="Number of concurrent threads to use for "
149 "creating payloads into the output globaltag.")
150 return
151
152 if not args.dry_run:
153 set_cdb_authentication_token(db, args.auth_token)
154
155 # prepare some colors for easy distinction of source tag
156 support_color = LogPythonInterface.terminal_supports_colors()
157 if support_color:
158 colors = "\x1b[32m \x1b[34m \x1b[35m \x1b[31m".split()
159 colors = {tag: color for tag, color in zip(args.globaltag, colors)}
160
161 def color_row(row, _, line):
162 """Color the lines depending on which globaltag the payload comes from"""
163 if not support_color:
164 return line
165 begin = colors.get(row[-1], "")
166 end = '\x1b[0m'
167 return begin + line + end
168
169 with Pager("Result of merging globaltags", True):
170 # make sure output tag exists
171 output_id = db.get_globalTagInfo(args.output)
172 if output_id is None:
173 B2ERROR("Output globaltag doesn't exist. Please create it first with a proper description")
174 return False
175
176 output_id = output_id["globalTagId"]
177
178 # check all globaltags exist
179 if any(db.get_globalTagInfo(tag) is None for tag in args.globaltag):
180 return False
181
182 final = []
183 table = []
184 existing = defaultdict(lambda: IoVSet(allow_overlaps=True))
185 if args.run_range is not None:
186 args.run_range = IntervalOfValidity(args.run_range)
187
188 # For each globaltag
189 for tag in args.globaltag:
190 # get all the payloads and iovs from the globaltag
191 all_payloads = db.get_all_iovs(tag)
192 # sort all the payloads by revision number (reversed sort: highest revisions first)
193 all_payloads.sort(key=lambda p: p.revision, reverse=True)
194 # and sort again but this time by name: not really necessary,
195 # but it helps printing the log messages ordered by payloads name
196 all_payloads.sort(key=lambda p: p.name, reverse=False)
197 # get all payload information objects with their iovs already merged to iovset instances
198 payloads = get_all_iovsets(all_payloads, args.run_range)
199 for payload in payloads:
200 # make sure it doesn't overlap with any of the previous
201 payload.iov.remove(existing[payload.name])
202 # but if there's something left
203 if payload.iov:
204 # extend the known coverage of this payload
205 existing[payload.name] |= payload.iov
206 # extend this payload to the list to create later
207 final.append(payload)
208 # and add all separate iovs to the table to show the user
209 for iov in payload.iov:
210 table.append([payload.name, payload.revision] + list(iov.tuple) + [tag])
211
212 # sort the table by payload name and start run ... we want to display it
213 table.sort(key=lambda r: (r[0], r[3:5]))
214
215 # and fancy print it ...
216 table.insert(0, ["Name", "Rev", "First Exp", "First Run", "Final Exp", "Final Run", "Source"])
217 columns = ["+", -8, 6, 6, 6, 6, max(len(_) for _ in args.globaltag)]
218
219 B2INFO(f"Result of merging the globaltags {', '.join(args.globaltag)}")
220
221 pretty_print_table(table, columns, transform=color_row)
222
223 # Ok, we're still alive, create all the payloads using multiple processes.
224 if not args.dry_run:
225 B2INFO(f'Now copying the {len(final)} payloads into {args.output} to create {len(table)-1} iovs ...')
226 create_iov = functools.partial(create_iov_wrapper, db, output_id)
227 try:
228 with ThreadPoolExecutor(max_workers=args.nprocess) as pool:
229 start = time.monotonic()
230 for payload, _ in enumerate(pool.map(create_iov, final), 1):
231 eta = (time.monotonic() - start) / payload * (len(final) - payload)
232 B2INFO(f"{payload}/{len(final)} payloads copied, ETA: {eta:.1f} seconds")
233 except RuntimeError:
234 B2ERROR("Not all iovs could be created. This could be a server/network problem "
235 "or the destination globaltag was not empty or not writeable. Please make "
236 "sure the target tag is empty and try again")
237 return 1
238
239 return 0
240
241
242def command_tag_runningupdate(args, db=None):
243 """
244 Update a running globaltag with payloads from a staging tag
245
246 This command will calculate and apply the necessary updates to a running
247 globaltag with a given staging globaltag
248
249 Running tags are defined as "immutable for existing data but conditions for newer runs may be added" and the only modification allowed is to add new
250 payloads for new runs or close existing payloads to no longer be valid for
251 new runs.
252
253 This command takes previously prepared and validated payloads in a staging
254 globaltag and will then calculate which payloads to close and what to add to
255 the running globaltag.
256
257 For this to work we require
258
259 1. A running globaltag in the state "RUNNING"
260
261 2. A (experiment, run) number from which run on the update should be valid.
262 This run number needs to be
263
264 a) bigger than the start of validity for all iovs in the running tag
265 b) bigger than the end of validity for all closed iovs (i.e. not valid
266 to infinity) in the running tag
267
268 3. A staging globaltag with the new payloads in state "VALIDATED"
269
270 a) payloads in the staging tag starting at (0,0) will be interpreted as
271 starting at the first valid run for the update
272 b) all other payloads need to start at or after the first valid run for
273 the update.
274 c) The globaltag needs to be gap and overlap free
275 d) All payloads in the staging tag should have as last iov an open iov
276 (i.e. valid to infinity) but this can be disabled.
277
278 The script will check all the above requirements and will then calculate the
279 necessary operations to
280
281 1. Add all payloads from the staging tag where a start validity of (0, 0) is
282 replaced by the starting run for which this update should be valid.
283 2. close all iovs for payloads in the running tags just before the
284 corresponding iov of the same payload in the staging tag, so either at the
285 first run for the update to be valid or later
286 3. Optionally, make sure all payloads in the staging tag end in an open iov.
287
288 .. rubric:: Examples
289
290 Running tag contains ::
291
292 payload1, rev 1, valid from 0,1 to 1,0
293 payload1, rev 2, valid from 1,1 to -1,-1
294 payload2, rev 1, valid from 0,1 to -1,-1
295 payload3, rev 1, valid from 0,1 to 1,0
296 payload4, rev 1, valid from 0,1 to -1,-1
297 payload5, rev 1, valid from 0,1 to -1,-1
298
299 Staging tag contains ::
300
301 payload1, rev 3, valid from 0,0 to 1,8
302 payload1, rev 4, valid from 1,9 to 1,20
303 payload2, rev 2, valid from 1,5 to 1,20
304 payload3, rev 2, valid from 0,0 to -1,-1
305 payload4, rev 1, valid from 0,0 to 1,20
306
307 Then running ``b2conditionsdb tag runningupdate running staging --run 1 2 --allow-closed``,
308 the running globaltag after the update will contain ::
309
310 payload1, rev 1, valid from 0,1 to 1,0
311 payload1, rev 2, valid from 1,1 to 1,1
312 payload1, rev 3, valid from 1,2 to 1,8
313 payload1, rev 4, valid from 1,9 to 1,20
314 payload2, rev 1, valid from 0,1 to 1,4
315 payload2, rev 2, valid from 1,5 to 1,20
316 payload3, rev 1, valid from 0,1 to 1,0
317 payload3, rev 2, valid from 1,2 to -1,-1
318 payload4, rev 1, valid from 0,1 to 1,20
319 payload5, rev 1, valid from 0,1 to -1,-1
320
321 Note that
322
323 - the start of payload1 and payload3 in staging has been adjusted
324
325 - payload2 in the running tag as been closed at 1,4, just before the
326 validity from the staging tag
327
328 - payload3 was already closed in the running tag so no change is
329 performed. This might result in gaps but is intentional
330
331 - payload4 was not closed at rim 1,2 because the staging tag had the same
332 revision of the payload so the these were merged to one long validity.
333
334 - payload5 was not closed as there was no update to it in the staging tag.
335 If we would have run with ``--full-replacement`` it would have been closed.
336
337 - if we would have chosen ``--run 1 1`` the update would have failed because
338 payload1, rev2 in running starts at 1,1 so we would have a conflict
339
340 - if we would have chosen ``--run 1 6`` the update would have failed because
341 payload2 in the staging tag starts before this run
342
343 - if we would have chosen to open the final iovs in staging by using
344 ``--fix-closed``, payload1, rev 4; payload2, rev 2 and payload4 rev 1
345 would be valid until -1,-1 after the running tag. In fact, payload 4
346 would not be changed at all.
347 """
348 if db is None:
349 args.add_argument("running", help="name of the running globaltag")
350 args.add_argument("staging", help="name of the staging globaltag")
351 group = args.add_argument_group("required named arguments")
352 group.add_argument("-r", "--run", required=True, nargs=2, type=int, metavar=("EXP", "RUN"),
353 help="First experiment + run number for which the update should be "
354 "valid. Two numbers separated by space")
355 choice = args.add_mutually_exclusive_group()
356 choice.add_argument("--allow-closed", dest="mode", action="store_const",
357 const=RunningTagUpdateMode.ALLOW_CLOSED,
358 default=RunningTagUpdateMode.STRICT,
359 help="if given allow payloads in the staging tag to not "
360 "be open, i.e. they don't have to be open ended in the "
361 "update. Useful to retire a payload by adding one last update")
362 choice.add_argument("--fix-closed", dest="mode", action="store_const",
363 const=RunningTagUpdateMode.FIX_CLOSED,
364 help="if given automatically open the last iov for each "
365 "payload in staging if it is closed.")
366 choice.add_argument("--simple-mode", dest="mode", action="store_const",
367 const=RunningTagUpdateMode.SIMPLE,
368 help="if given require the staging tag to solely consist "
369 "of fully infinite validities: Only one iov per payload "
370 "with a validity of (0,0,-1,-1)")
371 choice.add_argument("--full-replacement", dest="mode", action="store_const",
372 const=RunningTagUpdateMode.FULL_REPLACEMENT,
373 help="if given perform a full replacement and close all "
374 "open iovs in the running tag not present in the staging tag. "
375 "After such an update exactly the payloads in the staging tag "
376 "will be valid after the given run. This allows for closed iovs "
377 "in the staging tag as with ``--allow-closed``")
378 args.add_argument("--dry-run", default=False, action="store_true",
379 help="Only show the changes, don't try to apply them")
380 return
381
382 if not args.dry_run:
383 set_cdb_authentication_token(db, args.auth_token)
384
385 try:
386 updater = RunningTagUpdater(db, args.running, args.staging, args.run, args.mode, args.dry_run)
387 operations = updater.calculate_update()
388 except RunningTagUpdaterError as e:
389 B2ERROR(e, **e.extra_vars)
390 return 1
391
392 # make sure we exit if we have nothing to do
393 if not operations:
394 B2INFO("Nothing to do, please check the globaltags given are correct")
395 return 1
396
397 # show operations in a table and some summary
398 table = []
399 last_valid = tuple(args.run)
400 summary = {
401 "first valid run": last_valid,
402 "payloads closed": 0,
403 "payloads updated": 0,
404 "payload iovs added": 0,
405 "next possible update": last_valid,
406 }
407
408 updated_payloads = set()
409 for op, payload in operations:
410 # calculate how many payloads/iovs will be closed/added
411 if op == "CLOSE":
412 summary['payloads closed'] += 1
413 else:
414 updated_payloads.add(payload.name)
415 summary['payload iovs added'] += 1
416 # remember the highest run number of any iov, so first run
417 last_valid = max(payload.iov[:2], last_valid)
418 # and final run if not open
419 if payload.iov[2:] != (-1, -1):
420 last_valid = max(payload.iov[2:], last_valid)
421 # and add to the table of operations to be shown
422 table.append([op, payload.name, payload.revision] + list(payload.iov))
423
424 # calculate the next fee run
425 summary['next possible update'] = (last_valid[0] + (1 if last_valid[1] < 0 else 0), last_valid[1] + 1)
426 # and the number of distinct payloads
427 summary['payloads updated'] = len(updated_payloads)
428
429 # prepare some colors for easy distinction of closed payloads
430 support_color = LogPythonInterface.terminal_supports_colors()
431
432 def color_row(row, _, line):
433 """Color the lines depending on which globaltag the payload comes from"""
434 if not support_color:
435 return line
436 begin = "" if row[0] != "CLOSE" else "\x1b[31m"
437 end = '\x1b[0m'
438 return begin + line + end
439
440 # and then show the table
441 table.sort(key=lambda x: (x[1], x[3:], x[2], x[0]))
442 table.insert(0, ["Action", "Payload", "Rev", "First Exp", "First Run", "Final Exp", "Final Run"])
443 columns = [6, '*', -6, 6, 6, 6, 6]
444
445 with Pager(f"Changes to running tag {args.running}:", True):
446 B2INFO(f"Changes to be applied to the running tag {args.running}")
447 pretty_print_table(table, columns, transform=color_row)
448
449 if args.dry_run:
450 B2INFO("Running in dry mode, not applying any changes.", **summary)
451 return 0
452
453 B2WARNING("Applying these changes cannot be undone and further updates to "
454 "this run range will **NOT** be possible", **summary)
455 # ask if the user really knows what they're doing
456 answer = input("Are you sure you want to continue? [yes/No]: ")
457 while answer.lower().strip() not in ['yes', 'no', 'n', '']:
458 answer = input("Please enter 'yes' or 'no': ")
459
460 if answer.lower().strip() != 'yes':
461 B2INFO("Aborted by user ...")
462 return 1
463
464 # Ok, all set ... apply the update
465 try:
466 updater.apply_update()
467 B2INFO("done")
468 return 0
469 except RunningTagUpdaterError as e:
470 B2ERROR(e, **e.extra_vars)
471 return 1
472
473
474def command_iovs(args, db=None):
475 """
476 Modify, delete or copy iovs from a globaltags.
477
478 This command allows to modify, delete or copy iovs of a globaltag. If no other command is given do nothing.
479 """
480
481 # no arguments to define, just a group command
482 pass
483
484# Helper class to unify common parts of commands_iovs_*
485
486
487class CommandIoVsHelper:
488 """
489 Class to unify common parts of b2conditionsdb iovs commands
490
491 This class defines common argparse arguments,
492 common filter of iovs and common multithreading
493 """
494
495 def __init__(self, whichcommand, args, db):
496 """initialization, just remember the arguments or parser and the database instance
497
498 Args:
499 whichcommand (str): from whichcommand it is called (copy, delete or modify)
500 args (argparse.ArgumentParser): where to append new arguments
501 db (conditions_db.ConditionsDB): database instance to be used
502 """
503
504
505 self.whichcommand = whichcommand
507 self._args = args
509 self.db = db
511 self.iovfilter = ItemFilter(args)
513 self.num_all_iovs = None
515 self.past_dict = {"delete": "deleted", "modify": "modified", "copy": "copied", "create": "created"}
517 def add_arguments(self):
518 """Add arguments to the parser"""
519
520 self._args.add_argument("tag", metavar="INPUT_TAGNAME",
521 help=f"globaltag for which the the IoVs should be {self.past_dict[self.whichcommand]}")
522 if self.whichcommand == "copy":
523 self._args.add_argument("output", metavar="OUTPUT_TAGNAME", help="globaltag to which the the IoVs should be copied")
524 if self.whichcommand == "modify":
525 self._args.add_argument("new_iov", metavar="NEW_IOV", help="New iov to be set to all considered iovs."
526 " It should be a string with 4 numbers separated by spaces."
527 " Use * to mark the fields that should not be modified. For example"
528 " if 7 0 * * is given the iov (7, 1, 9, 42) will become (7 0 9 42).")
529 self._args.add_argument("--iov-id", default=None, type=int,
530 help="IoVid of the iov to be considered")
531 self._args.add_argument(
532 "--iov-pattern",
533 default=None,
534 help="whitespace-separated string with pattern of the iov to be considered. "
535 " Use * to mark the fields that shold be ignored. Valid patterns are 0 0 -1 -1"
536 " (a very specific IoV), 0 * -1 -1 (any iov that starts in any run of exp 0 and ends exactly in exp -1, run -1)"
537 ", * * 3 45 (any Iov ending in exp 3, run 45, regardless from where it starts).")
538 self._args.add_argument("--run-range", nargs=4, default=None, type=int,
539 metavar=("FIRST_EXP", "FIRST_RUN", "FINAL_EXP", "FINAL_RUN"),
540 help="Can be four numbers to limit the run range to be considered"
541 " Only iovs overlapping, even partially, with this range will be considered.")
542 self._args.add_argument("--fully-contained", action="store_true",
543 help="If given together with ``--run_range`` limit the list of payloads "
544 "to the ones fully contained in the given run range")
545 if self.whichcommand == "copy":
546 self._args.add_argument("--set-run-range", action="store_true",
547 help="If given together with ``--run_range`` modify the interval of validity"
548 " of partially overlapping iovs to be fully contained in the given run range")
549 self.iovfilter.add_arguments("payloads")
550 self._args.add_argument("--revision", metavar='revision', type=int,
551 help="Specify the revision of the payload to be removed")
552 self._args.add_argument("--dry-run", help="Don't do anything, just print what would be done",
553 action="store_true", default=False)
554 if self.whichcommand == "copy":
555 self._args.add_argument("--replace", help="Modify the iovs in the output tag to avoid overlaps",
556 action="store_true", default=False)
557 self._args.add_argument("-j", type=int, default=10, dest="nprocess",
558 help="Number of concurrent threads to use.")
559
560 def get_iovs(self):
561 """Get the iovs already filtered"""
562 if not self.iovfilter.check_arguments():
563 B2ERROR("Issue with arguments")
564
565 all_iovs = self.db.get_all_iovs(
566 self._args.tag,
567 run_range=self._args.run_range,
568 fully_contained=self._args.fully_contained,
569 message=str(self.iovfilter))
570 self.num_all_iovs = len(all_iovs)
571 iovs_to_return = []
572 for iov in all_iovs:
573 if self._args.iov_id and iov.iov_id != self._args.iov_id:
574 continue
575 if not self.iovfilter.check(iov.name):
576 continue
577 if self._args.iov_pattern and not fnmatch("{} {} {} {}".format(*iov.iov),
578 "{} {} {} {}".format(*self._args.iov_pattern.split())):
579 continue
580 if self._args.revision and iov.revision != self._args.revision:
581 continue
582 iovs_to_return.append(iov)
583 return iovs_to_return
584
585 def modify_db(self, func, func_args, whichcommand=None):
586 """Modify the database using multithreading"""
587 if whichcommand is None:
588 whichcommand = self.whichcommand
589 try:
590 with ThreadPoolExecutor(max_workers=self._args.nprocess) as pool:
591 start = time.monotonic()
592 for iov_num, _ in enumerate(pool.map(func, func_args), 1):
593 eta = (time.monotonic() - start) / iov_num * (len(func_args) - iov_num)
594 B2INFO(f"{iov_num}/{len(func_args)} iovs {self.past_dict[whichcommand]}, ETA: {eta:.1f} seconds")
595 except RuntimeError:
596 B2ERROR(f"Not all iovs could be {self.past_dict[whichcommand]}. This could be a server/network problem "
597 "or the destination globaltag was not writeable.")
598 raise
599
600
601def command_iovs_delete(args, db=None):
602 """
603 Delete iovs from a globaltag
604
605 This command allows to delete the iovs from a globaltags, optionally limiting the iovs to be deleted to those
606 of a specific payload, revision, IoVid or run range.
607 """
608 command_iovs_helper = CommandIoVsHelper("delete", args, db)
609 if db is None:
610 command_iovs_helper.add_arguments()
611 return
612
613 iovs_to_delete = command_iovs_helper.get_iovs()
614
615 table = [[i.iov_id, i.name, i.revision] + list(i.iov) for i in iovs_to_delete]
616 table.insert(0, ["IovId", "Name", "Rev", "First Exp", "First Run", "Final Exp", "Final Run"])
617 columns = [9, "+", 6, 6, 6, 6, 6]
618
619 if command_iovs_helper.num_all_iovs == len(iovs_to_delete) and args.run_range is None:
620 B2WARNING(f"All the iovs in the globaltag {args.tag} will be deleted!")
621 gt_check = input('Please enter the global tag name again to confirm this action: ')
622 if gt_check != args.tag:
623 B2ERROR('global tag names do not match.')
624 return 1
625
626 B2INFO("Deleting the following iovs")
627 pretty_print_table(table, columns)
628
629 if not args.dry_run:
630 set_cdb_authentication_token(db, args.auth_token)
631 try:
632 command_iovs_helper.modify_db(db.delete_iov, [i.iov_id for i in iovs_to_delete])
633 except RuntimeError:
634 return 1
635
636 return 0
637
638
639def command_iovs_copy(args, db=None):
640 """
641 Copy iovs from a globaltag to another one
642
643 This command allows to copy the iovs from a globaltags to another one, optionally limiting
644 the iovs to be copied to those of a specific payload, revision, IoV id or run range.
645 """
646 command_iovs_helper = CommandIoVsHelper("copy", args, db)
647 if db is None:
648 command_iovs_helper.add_arguments()
649 return
650
651 iovs_to_copy = command_iovs_helper.get_iovs()
652
653 # make sure that there are no overlaps in the iovs to copy and set run range if needed
654 by_name = defaultdict(lambda: IoVSet(allow_overlaps=False))
655 for iov in iovs_to_copy:
656 if args.run_range and args.set_run_range:
657 iov.iov = (IntervalOfValidity(*iov.iov) & IntervalOfValidity(*args.run_range)).tuple
658 try:
659 by_name[iov.name].add(iov.iov)
660 except ValueError as e:
661 B2ERROR(f"Overlap for payload {iov.name} r{iov.revision}: {e}")
662 return 1
663 del by_name
664
665 # make sure output tag exists
666 output_id = db.get_globalTagInfo(args.output)
667 if output_id is None:
668 B2ERROR("Output globaltag doesn't exist. Please create it first with a proper description")
669 return False
670 output_id = output_id["globalTagId"]
671
672 # get iovs already present in output global tag
673 iovs_output = db.get_all_iovs(args.output)
674 iovs_output.sort(key=lambda i: i.iov[:2])
675 payloads_output = defaultdict(list)
676 for iov in iovs_output:
677 iov.newiov = IoVSet([IntervalOfValidity(*iov.iov)])
678 payloads_output[iov.name].append(iov)
679
680 # remove from iovs to copy those that are already present in the output global tag
681 iovs_in_output = [(i.checksum, i.iov) for i in iovs_output]
682 iovs_to_copy = [i for i in iovs_to_copy if (i.checksum, i.iov) not in iovs_in_output]
683 del iovs_in_output
684
685 iovs_not_to_copy = []
686
687 # readjust the iovs already present if they overlap
688 # with the ones to be copied
689 for iov_to_copy in iovs_to_copy:
690 iov_ = IntervalOfValidity(*iov_to_copy.iov)
691 still_to_copy = True
692 for iov_present in payloads_output[iov_to_copy.name]:
693 if iov_present.newiov & iov_:
694 if still_to_copy and iov_to_copy.checksum == iov_present.checksum:
695 iov_present.newiov.add(iov_, allow_overlaps=True)
696 iovs_not_to_copy.append(iov_to_copy)
697 still_to_copy = False
698 else:
699 iov_present.newiov -= iov_
700
701 iovs_to_delete = []
702 iovs_to_modify = []
703 iovs_to_create = []
704
705 # Fill lists of iovs to be deleted, modified or added
706 for iov_present in sum(payloads_output.values(), []):
707 # iov_present was replaced by copied iovs so it should be deleted
708 if len(iov_present.newiov) == 0:
709 iovs_to_delete.append(iov_present)
710 # iov_present was partially overlapping by copied iovs so it should be modified
711 elif len(iov_present.newiov) == 1:
712 if not list(iov_present.newiov)[0].tuple == iov_present.iov:
713 iovs_to_modify.append((iov_present, list(iov_present.newiov)[0].tuple))
714 # iov_present was split by copied iovs so its first iov should be modified and new iovs should be created
715 else:
716 newiovs = list(iov_present.newiov)
717 iovs_to_modify.append((iov_present, newiovs[0].tuple))
718 for newiov in newiovs[1:]:
719 iovs_to_create.append((iov_present, newiov.tuple))
720
721 # If I fix the overlaps do not copy iovs that can be obtained extending iovs already present
722 if args.replace:
723 iovs_to_copy = [i for i in iovs_to_copy if i not in iovs_not_to_copy]
724
725 # Print nice table
726 table = [[i.iov_id, i.name, i.revision] + list(i.iov) for i in iovs_to_copy]
727 table.insert(0, ["IovId", "Name", "Rev", "First Exp", "First Run", "Final Exp", "Final Run"])
728 columns = [9, "+", 6, 6, 6, 6, 6]
729 B2INFO(f"Copying the following iovs to {args.output}")
730 pretty_print_table(table, columns)
731
732 if (iovs_to_delete or iovs_to_modify or iovs_to_create) and not args.replace:
733 B2WARNING("Inserting the iovs will create overlaps,"
734 f" to avoid them the changes below should be implemented on {args.tag}")
735
736 if iovs_to_delete:
737 B2WARNING("iovs to be deleted")
738 # Sort by payload name and iov for nicer logs
739 iovs_to_delete.sort(key=lambda x: x.iov[:2])
740 iovs_to_delete.sort(key=lambda x: x.name)
741 # Print nice table
742 table = [[i.iov_id, i.name, i.revision] + list(i.iov) for i in iovs_to_delete]
743 table.insert(0, ["IovId", "Name", "Rev", "First Exp", "First Run", "Final Exp", "Final Run"])
744 columns = [9, "+", 6, 6, 6, 6, 6]
745 pretty_print_table(table, columns)
746 if iovs_to_modify:
747 B2WARNING("iovs to be modified")
748 # Sort by payload name and iov for nicer logs
749 iovs_to_modify.sort(key=lambda x: x[0].iov[:2])
750 iovs_to_modify.sort(key=lambda x: x[0].name)
751 # Print nice table
752 table = [[i[0].iov_id, i[0].name, i[0].revision] + list(i[0].iov) + list(i[1]) for i in iovs_to_modify]
753 table.insert(0, ["IovId", "Name", "Rev", "Old First Exp", "Old First Run", "Old Final Exp", "Old Final Run",
754 "New First Exp", "New First Run", "New Final Exp", "New Final Run"])
755 columns = [9, "+", 6, 6, 6, 6, 6, 6, 6, 6, 6]
756 pretty_print_table(table, columns)
757 if iovs_to_create:
758 B2WARNING("iovs to be created")
759 # Sort by payload name and iov for nicer logs
760 iovs_to_create.sort(key=lambda x: x[1][:2])
761 iovs_to_create.sort(key=lambda x: x[0].name)
762 # Print nice table
763 table = [[i[0].name, i[0].revision] + list(i[1]) for i in iovs_to_create]
764 table.insert(0, ["Name", "Rev", "First Exp", "First Run", "Final Exp", "Final Run"])
765 columns = ["+", 6, 6, 6, 6, 6]
766 pretty_print_table(table, columns)
767
768 if (iovs_to_delete or iovs_to_modify or iovs_to_create) and not args.replace:
769 B2WARNING("To apply them use the option --replace")
770 B2WARNING("If instead you want these overlaps to be present enter"
771 " the output global tag name again to confirm this action: ")
772 gt_check = input()
773 if gt_check != args.output:
774 B2ERROR("global tag names do not match.")
775 return 1
776
777 if not args.dry_run:
778 set_cdb_authentication_token(db, args.auth_token)
779 try:
780 command_iovs_helper.modify_db(lambda args: db.create_iov(output_id, *args),
781 [(i.payload_id, *i.iov) for i in iovs_to_copy])
782 if args.replace:
783 command_iovs_helper.modify_db(db.delete_iov, [i.iov_id for i in iovs_to_delete], "delete")
784 command_iovs_helper.modify_db(lambda args: db.modify_iov(*args),
785 [(i[0].iov_id, *i[1]) for i in iovs_to_modify], "modify")
786 command_iovs_helper.modify_db(lambda args: db.create_iov(output_id, *args),
787 [(i[0].payload_id, *i[1]) for i in iovs_to_create], "create")
788 except RuntimeError:
789 return 1
790
791 return 0
792
793
794def command_iovs_modify(args, db=None):
795 """
796 Modify iovs from a globaltag
797
798 This command allows to modify the iovs from a globaltags, optionally limiting the iovs to be modified to those
799 of a specific payload, revision, IoV id or run range.
800 """
801 command_iovs_helper = CommandIoVsHelper("modify", args, db)
802 if db is None:
803 command_iovs_helper.add_arguments()
804 return
805
806 new_iov = args.new_iov.split()
807 # Transform integer values of iov in int and keep '*' as strings. Raise error if something different is given.
808 for i in range(len(new_iov)):
809 try:
810 new_iov[i] = int(new_iov[i])
811 except ValueError:
812 if new_iov[i] != '*':
813 raise ValueError(f"Invalid IOV value: {new_iov[i]} should be an integer or '*'")
814
815 iovs_to_modify = []
816 for iov in command_iovs_helper.get_iovs():
817 new_iow_ = new_iov.copy()
818 for i in range(len(new_iow_)):
819 if new_iow_[i] == '*':
820 new_iow_[i] = iov.iov[i]
821 iovs_to_modify.append((iov, new_iow_))
822
823 table = [[i[0].iov_id, i[0].name, i[0].revision] + list(i[0].iov) + list(i[1]) for i in iovs_to_modify]
824 table.insert(0, ["IovId", "Name", "Rev", "Old First Exp", "Old First Run", "Old Final Exp", "Old Final Run",
825 "New First Exp", "New First Run", "New Final Exp", "New Final Run"])
826 columns = [9, "+", 6, 6, 6, 6, 6, 6, 6, 6, 6]
827
828 B2INFO("Changing the following iovs")
829 pretty_print_table(table, columns)
830
831 if not args.dry_run:
832 set_cdb_authentication_token(db, args.auth_token)
833 try:
834 command_iovs_helper.modify_db(lambda args: db.modify_iov(*args),
835 [(i[0].iov_id, *i[1]) for i in iovs_to_modify])
836 except RuntimeError:
837 return 1
838
839 return 0
840
_args
argparse.ArgumentParser instance
past_dict
Dictionary with past participles.
def modify_db(self, func, func_args, whichcommand=None)
db
conditions_db.ConditionsDB instance
num_all_iovs
number of iovs before payload and revision selection
def __init__(self, whichcommand, args, db)
whichcommand
from whichcommand it is called (copy, delete or modify)