Belle II Software development
cli_management.py
1
8import functools
9import time
10from fnmatch import fnmatch
11from concurrent.futures import ThreadPoolExecutor
12from collections import defaultdict
13from basf2 import B2INFO, B2ERROR, B2WARNING, LogPythonInterface # noqa
14from basf2.utils import pretty_print_table
15from terminal_utils import Pager
16from conditions_db import set_cdb_authentication_token
17from conditions_db.iov import IoVSet, IntervalOfValidity
18from conditions_db.runningupdate import RunningTagUpdater, RunningTagUpdaterError, RunningTagUpdateMode
19from conditions_db.cli_utils import ItemFilter
20
21
22def get_all_iovsets(existing_payloads, run_range=None):
23 """Given a list of PayloadInformation objects, return a reduced list PayloadInformation
24 objects with the single iovs replaced with IoVSets. Payloads with the same
25 name and revision will be merged.
26
27 Overlaps will raise an B2ERROR
28 """
29 all_payloads = defaultdict(lambda: IoVSet(allow_overlaps=True))
30 by_name = defaultdict(lambda: IoVSet(allow_overlaps=False))
31 infos = {}
32 for payload in existing_payloads:
33 # we want to make set of iovs for each payload
34 iov = IntervalOfValidity(payload.iov)
35 # possibly we have a run range we want to limit to
36 if run_range is not None:
37 iov &= run_range
38 if not iov:
39 continue
40
41 # merge the iovs for the same revision
42 all_payloads[payload.name, payload.revision].add(iov)
43 # also keep the PayloadInformation
44 infos[payload.name, payload.revision] = payload
45
46 # and also check if there are any overlaps with the same payload name
47 try:
48 by_name[payload.name].add(iov)
49 except ValueError as e:
50 B2ERROR(f"Overlap for payload {payload.name} r{payload.revision}: {e}")
51
52 # so now flatten the thing again and return PayloadInformation objects we slightly modify
53 result = []
54 for (name, revision), iov in all_payloads.items():
55 info = infos[name, revision]
56 info.iov = iov
57 info.iov_id = None
58 result.append(info)
59
60 return result
61
62
63def create_iov_wrapper(db, globaltag_id, payload):
64 """
65 Wrapper function for adding payloads into a given globaltag.
66 """
67 for iov in payload.iov:
68 if db.create_iov(globaltag_id, payload.payload_id, *iov.tuple) is None:
69 raise RuntimeError(f"Cannot create iov for {payload.name} r{payload.revision}")
70
71
72def command_tag_merge(args, db=None):
73 """
74 Merge a list of globaltags in the order they are given.
75
76 This command allows to merge a number of globaltags into a single globaltag.
77 Payloads from later globaltags in the list of arguments are used to fill gaps
78 present in earlier globaltags.
79
80 The result is equivalent to having multiple globaltags setup in the conditions
81 access for basf2 (highest priority goes first).
82
83 .. warning::
84
85 The order of the globaltags is highest priority first, so payloads from
86 globaltags earlier on the command line will be taken with before globaltags
87 from later tags.
88
89 For each globaltag in the list we copy all payloads to the output globaltag
90 if there is no payload of that name valid for the given interval of validity
91 in any previous globaltags in the list.
92
93 If the payload overlaps partially with a payload from a previous globaltag
94 in the list the interval of validity is shortened (and possibly split) to
95 not overlap but to just fill the gaps.
96
97 For example:
98
99 Globaltag ``A`` contains ::
100
101 payload1, rev 2, valid from 1,0 to 1,10
102 payload1, rev 3, valid from 1,20 to 1,22
103 payload2, rev 1, valid from 1,0 to 1,-1
104
105 Globaltag ``B`` contains ::
106
107 payload1, rev 1, valid from 1,1 to 1,30
108 payload2, rev 2, valid from 0,1 to 1,20
109
110 Then running ``b2conditionsdb tag merge -o C A B``, the output globaltag ``C``
111 after the merge will contain::
112
113 payload1, rev 2, valid from 1,0 to 1,10
114 payload1, rev 1, valid from 1,11 to 1,19
115 payload1, rev 3, valid from 1,20 to 1,22
116 payload1, rev 1, valid from 1,23 to 1,30
117 payload2, rev 2, valid from 0,1 to 0,-1
118 payload2, rev 1, valid from 1,0 to 1,-1
119
120 When finished, this command will print a table of payloads and their
121 validity and from which globaltag they were taken. If ``--dry-run`` is given
122 it will only print the list of payloads.
123
124 Optionally one can specify ``--run-range`` to limit the run range for which
125 the merging should take place. In the example above, running with
126 ``--run-range 1 0 1 21`` the result would be ::
127
128 payload1, rev 2, valid from 1,0 to 1,10
129 payload1, rev 1, valid from 1,11 to 1,19
130 payload1, rev 3, valid from 1,20 to 1,21
131 payload2, rev 1, valid from 1,0 to 1,21
132
133 .. note:: Version added: release-05-01-00
134 """
135
136 if db is None:
137 args.add_argument("globaltag", nargs="+", help="name of the globaltag")
138 group = args.add_argument_group("required named arguments")
139 group.add_argument("-o", "--output", required=True, help="Name of the output globaltag")
140 args.add_argument("--dry-run", help="Don't do anything, just print a table with the results",
141 action="store_true", default=False)
142 args.add_argument("--run-range", nargs=4, default=None, type=int,
143 metavar=("FIRST_EXP", "FIRST_RUN", "FINAL_EXP", "FINAL_RUN"),
144 help="Can be for numbers to limit the run range to put"
145 "in the output globaltag: All iovs will be limited to "
146 "be in this range.")
147 args.add_argument("-j", type=int, default=10, dest="nprocess",
148 help="Number of concurrent threads to use for "
149 "creating payloads into the output globaltag.")
150 return
151
152 if not args.dry_run:
153 set_cdb_authentication_token(db, args.auth_token)
154
155 # prepare some colors for easy distinction of source tag
156 support_color = LogPythonInterface.terminal_supports_colors()
157 if support_color:
158 colors = "\x1b[32m \x1b[34m \x1b[35m \x1b[31m".split()
159 colors = {tag: color for tag, color in zip(args.globaltag, colors)}
160
161 def color_row(row, _, line):
162 """Color the lines depending on which globaltag the payload comes from"""
163 if not support_color:
164 return line
165 begin = colors.get(row[-1], "")
166 end = '\x1b[0m'
167 return begin + line + end
168
169 with Pager("Result of merging globaltags", True):
170 # make sure output tag exists
171 output_id = db.get_globalTagInfo(args.output)
172 if output_id is None:
173 B2ERROR("Output globaltag doesn't exist. Please create it first with a proper description")
174 return False
175
176 output_id = output_id["globalTagId"]
177
178 # check all globaltags exist
179 if any(db.get_globalTagInfo(tag) is None for tag in args.globaltag):
180 return False
181
182 final = []
183 table = []
184 existing = defaultdict(lambda: IoVSet(allow_overlaps=True))
185 if args.run_range is not None:
186 args.run_range = IntervalOfValidity(args.run_range)
187
188 # For each globaltag
189 for tag in args.globaltag:
190 # get all the payloads and iovs from the globaltag
191 all_payloads = db.get_all_iovs(tag)
192 # sort all the payloads by revision number (reversed sort: highest revisions first)
193 all_payloads.sort(key=lambda p: p.revision, reverse=True)
194 # and sort again but this time by name: not really necessary,
195 # but it helps printing the log messages ordered by payloads name
196 all_payloads.sort(key=lambda p: p.name, reverse=False)
197 # get all payload information objects with their iovs already merged to iovset instances
198 payloads = get_all_iovsets(all_payloads, args.run_range)
199 for payload in payloads:
200 # make sure it doesn't overlap with any of the previous
201 payload.iov.remove(existing[payload.name])
202 # but if there's something left
203 if payload.iov:
204 # extend the known coverage of this payload
205 existing[payload.name] |= payload.iov
206 # extend this payload to the list to create later
207 final.append(payload)
208 # and add all separate iovs to the table to show the user
209 for iov in payload.iov:
210 table.append([payload.name, payload.revision] + list(iov.tuple) + [tag])
211
212 # sort the table by payload name and start run ... we want to display it
213 table.sort(key=lambda r: (r[0], r[3:5]))
214
215 # and fancy print it ...
216 table.insert(0, ["Name", "Rev", "First Exp", "First Run", "Final Exp", "Final Run", "Source"])
217 columns = ["+", -8, 6, 6, 6, 6, max(len(_) for _ in args.globaltag)]
218
219 B2INFO(f"Result of merging the globaltags {', '.join(args.globaltag)}")
220
221 pretty_print_table(table, columns, transform=color_row)
222
223 # Ok, we're still alive, create all the payloads using multiple processes.
224 if not args.dry_run:
225 B2INFO(f'Now copying the {len(final)} payloads into {args.output} to create {len(table)-1} iovs ...')
226 create_iov = functools.partial(create_iov_wrapper, db, output_id)
227 try:
228 with ThreadPoolExecutor(max_workers=args.nprocess) as pool:
229 start = time.monotonic()
230 for payload, _ in enumerate(pool.map(create_iov, final), 1):
231 eta = (time.monotonic() - start) / payload * (len(final) - payload)
232 B2INFO(f"{payload}/{len(final)} payloads copied, ETA: {eta:.1f} seconds")
233 except RuntimeError:
234 B2ERROR("Not all iovs could be created. This could be a server/network problem "
235 "or the destination globaltag was not empty or not writeable. Please make "
236 "sure the target tag is empty and try again")
237 return 1
238
239 return 0
240
241
242def command_tag_runningupdate(args, db=None):
243 """
244 Update a running globaltag with payloads from a staging tag
245
246 This command will calculate and apply the necessary updates to a running
247 globaltag with a given staging globaltag
248
249 Running tags are defined as "immutable for existing data but conditions for
250 newer runs may be added" and the only modification allowed is to add new
251 payloads for new runs or close existing payloads to no longer be valid for
252 new runs.
253
254 This command takes previously prepared and validated payloads in a staging
255 globaltag and will then calculate which payloads to close and what to add to
256 the running globaltag.
257
258 For this to work we require
259
260 1. A running globaltag in the state "RUNNING"
261
262 2. A (experiment, run) number from which run on the update should be valid.
263 This run number needs to be
264
265 a) bigger than the start of validity for all iovs in the running tag
266 b) bigger than the end of validity for all closed iovs (i.e. not valid
267 to infinity) in the running tag
268
269 3. A staging globaltag with the new payloads in state "VALIDATED"
270
271 a) payloads in the staging tag starting at (0,0) will be interpreted as
272 starting at the first valid run for the update
273 b) all other payloads need to start at or after the first valid run for
274 the update.
275 c) The globaltag needs to be gap and overlap free
276 d) All payloads in the staging tag should have as last iov an open iov
277 (i.e. valid to infinity) but this can be disabled.
278
279 The script will check all the above requirements and will then calculate the
280 necessary operations to
281
282 1. Add all payloads from the staging tag where a start validity of (0, 0) is
283 replaced by the starting run for which this update should be valid.
284 2. close all iovs for payloads in the running tags just before the
285 corresponding iov of the same payload in the staging tag, so either at the
286 first run for the update to be valid or later
287 3. Optionally, make sure all payloads in the staging tag end in an open iov.
288
289 .. rubric:: Examples
290
291 Running tag contains ::
292
293 payload1, rev 1, valid from 0,1 to 1,0
294 payload1, rev 2, valid from 1,1 to -1,-1
295 payload2, rev 1, valid from 0,1 to -1,-1
296 payload3, rev 1, valid from 0,1 to 1,0
297 payload4, rev 1, valid from 0,1 to -1,-1
298 payload5, rev 1, valid from 0,1 to -1,-1
299
300 Staging tag contains ::
301
302 payload1, rev 3, valid from 0,0 to 1,8
303 payload1, rev 4, valid from 1,9 to 1,20
304 payload2, rev 2, valid from 1,5 to 1,20
305 payload3, rev 2, valid from 0,0 to -1,-1
306 payload4, rev 1, valid from 0,0 to 1,20
307
308 Then running ``b2conditionsdb tag runningupdate running staging --run 1 2 --allow-closed``,
309 the running globaltag after the update will contain ::
310
311 payload1, rev 1, valid from 0,1 to 1,0
312 payload1, rev 2, valid from 1,1 to 1,1
313 payload1, rev 3, valid from 1,2 to 1,8
314 payload1, rev 4, valid from 1,9 to 1,20
315 payload2, rev 1, valid from 0,1 to 1,4
316 payload2, rev 2, valid from 1,5 to 1,20
317 payload3, rev 1, valid from 0,1 to 1,0
318 payload3, rev 2, valid from 1,2 to -1,-1
319 payload4, rev 1, valid from 0,1 to 1,20
320 payload5, rev 1, valid from 0,1 to -1,-1
321
322 Note that
323
324 - the start of payload1 and payload3 in staging has been adjusted
325
326 - payload2 in the running tag as been closed at 1,4, just before the
327 validity from the staging tag
328
329 - payload3 was already closed in the running tag so no change is
330 performed. This might result in gaps but is intentional
331
332 - payload4 was not closed at rim 1,2 because the staging tag had the same
333 revision of the payload so the these were merged to one long validity.
334
335 - payload5 was not closed as there was no update to it in the staging tag.
336 If we would have run with ``--full-replacement`` it would have been closed.
337
338 - if we would have chosen ``--run 1 1`` the update would have failed because
339 payload1, rev2 in running starts at 1,1 so we would have a conflict
340
341 - if we would have chosen ``--run 1 6`` the update would have failed because
342 payload2 in the staging tag starts before this run
343
344 - if we would have chosen to open the final iovs in staging by using
345 ``--fix-closed``, payload1, rev 4; payload2, rev 2 and payload4 rev 1
346 would be valid until -1,-1 after the running tag. In fact, payload 4
347 would not be changed at all.
348 """
349 if db is None:
350 args.add_argument("running", help="name of the running globaltag")
351 args.add_argument("staging", help="name of the staging globaltag")
352 group = args.add_argument_group("required named arguments")
353 group.add_argument("-r", "--run", required=True, nargs=2, type=int, metavar=("EXP", "RUN"),
354 help="First experiment + run number for which the update should be "
355 "valid. Two numbers separated by space")
356 choice = args.add_mutually_exclusive_group()
357 choice.add_argument("--allow-closed", dest="mode", action="store_const",
358 const=RunningTagUpdateMode.ALLOW_CLOSED,
359 default=RunningTagUpdateMode.STRICT,
360 help="if given allow payloads in the staging tag to not "
361 "be open, i.e. they don't have to be open ended in the "
362 "update. Useful to retire a payload by adding one last update")
363 choice.add_argument("--fix-closed", dest="mode", action="store_const",
364 const=RunningTagUpdateMode.FIX_CLOSED,
365 help="if given automatically open the last iov for each "
366 "payload in staging if it is closed.")
367 choice.add_argument("--simple-mode", dest="mode", action="store_const",
368 const=RunningTagUpdateMode.SIMPLE,
369 help="if given require the staging tag to solely consist "
370 "of fully infinite validities: Only one iov per payload "
371 "with a validity of (0,0,-1,-1)")
372 choice.add_argument("--full-replacement", dest="mode", action="store_const",
373 const=RunningTagUpdateMode.FULL_REPLACEMENT,
374 help="if given perform a full replacement and close all "
375 "open iovs in the running tag not present in the staging tag. "
376 "After such an update exactly the payloads in the staging tag "
377 "will be valid after the given run. This allows for closed iovs "
378 "in the staging tag as with ``--allow-closed``")
379 args.add_argument("--dry-run", default=False, action="store_true",
380 help="Only show the changes, don't try to apply them")
381 return
382
383 if not args.dry_run:
384 set_cdb_authentication_token(db, args.auth_token)
385
386 try:
387 updater = RunningTagUpdater(db, args.running, args.staging, args.run, args.mode, args.dry_run)
388 operations = updater.calculate_update()
389 except RunningTagUpdaterError as e:
390 B2ERROR(e, **e.extra_vars)
391 return 1
392
393 # make sure we exit if we have nothing to do
394 if not operations:
395 B2INFO("Nothing to do, please check the globaltags given are correct")
396 return 1
397
398 # show operations in a table and some summary
399 table = []
400 last_valid = tuple(args.run)
401 summary = {
402 "first valid run": last_valid,
403 "payloads closed": 0,
404 "payloads updated": 0,
405 "payload iovs added": 0,
406 "next possible update": last_valid,
407 }
408
409 updated_payloads = set()
410 for op, payload in operations:
411 # calculate how many payloads/iovs will be closed/added
412 if op == "CLOSE":
413 summary['payloads closed'] += 1
414 else:
415 updated_payloads.add(payload.name)
416 summary['payload iovs added'] += 1
417 # remember the highest run number of any iov, so first run
418 last_valid = max(payload.iov[:2], last_valid)
419 # and final run if not open
420 if payload.iov[2:] != (-1, -1):
421 last_valid = max(payload.iov[2:], last_valid)
422 # and add to the table of operations to be shown
423 table.append([op, payload.name, payload.revision] + list(payload.iov))
424
425 # calculate the next fee run
426 summary['next possible update'] = (last_valid[0] + (1 if last_valid[1] < 0 else 0), last_valid[1] + 1)
427 # and the number of distinct payloads
428 summary['payloads updated'] = len(updated_payloads)
429
430 # prepare some colors for easy distinction of closed payloads
431 support_color = LogPythonInterface.terminal_supports_colors()
432
433 def color_row(row, _, line):
434 """Color the lines depending on which globaltag the payload comes from"""
435 if not support_color:
436 return line
437 begin = "" if row[0] != "CLOSE" else "\x1b[31m"
438 end = '\x1b[0m'
439 return begin + line + end
440
441 # and then show the table
442 table.sort(key=lambda x: (x[1], x[3:], x[2], x[0]))
443 table.insert(0, ["Action", "Payload", "Rev", "First Exp", "First Run", "Final Exp", "Final Run"])
444 columns = [6, '*', -6, 6, 6, 6, 6]
445
446 with Pager(f"Changes to running tag {args.running}:", True):
447 B2INFO(f"Changes to be applied to the running tag {args.running}")
448 pretty_print_table(table, columns, transform=color_row)
449
450 if args.dry_run:
451 B2INFO("Running in dry mode, not applying any changes.", **summary)
452 return 0
453
454 B2WARNING("Applying these changes cannot be undone and further updates to "
455 "this run range will **NOT** be possible", **summary)
456 # ask if the user really knows what they're doing
457 answer = input("Are you sure you want to continue? [yes/No]: ")
458 while answer.lower().strip() not in ['yes', 'no', 'n', '']:
459 answer = input("Please enter 'yes' or 'no': ")
460
461 if answer.lower().strip() != 'yes':
462 B2INFO("Aborted by user ...")
463 return 1
464
465 # Ok, all set ... apply the update
466 try:
467 updater.apply_update()
468 B2INFO("done")
469 return 0
470 except RunningTagUpdaterError as e:
471 B2ERROR(e, **e.extra_vars)
472 return 1
473
474
475def command_iovs(args, db=None):
476 """
477 Modify, delete or copy iovs from a globaltags.
478
479 This command allows to modify, delete or copy iovs of a globaltag. If no other command is given do nothing.
480 """
481
482 # no arguments to define, just a group command
483 pass
484
485# Helper class to unify common parts of commands_iovs_*
486
487
488class CommandIoVsHelper:
489 """
490 Class to unify common parts of b2conditionsdb iovs commands
491
492 This class defines common argparse arguments,
493 common filter of iovs and common multithreading
494 """
495
496 def __init__(self, whichcommand, args, db):
497 """initialization, just remember the arguments or parser and the database instance
498
499 Args:
500 whichcommand (str): from whichcommand it is called (copy, delete or modify)
501 args (argparse.ArgumentParser): where to append new arguments
502 db (conditions_db.ConditionsDB): database instance to be used
503 """
504
505
506 self.whichcommand = whichcommand
507
508 self._args = args
509
510 self.db = db
511
513
514 self.num_all_iovs = None
515
516 self.past_dict = {"delete": "deleted", "modify": "modified", "copy": "copied", "create": "created"}
517
518 def add_arguments(self):
519 """Add arguments to the parser"""
520
521 self._args.add_argument("tag", metavar="INPUT_TAGNAME",
522 help=f"globaltag for which the the IoVs should be {self.past_dict[self.whichcommand]}")
523 if self.whichcommand == "copy":
524 self._args.add_argument("output", metavar="OUTPUT_TAGNAME", help="globaltag to which the the IoVs should be copied")
525 if self.whichcommand == "modify":
526 self._args.add_argument("new_iov", metavar="NEW_IOV", help="New iov to be set to all considered iovs."
527 " It should be a string with 4 numbers separated by spaces."
528 " Use * to mark the fields that should not be modified. For example"
529 " if 7 0 * * is given the iov (7, 1, 9, 42) will become (7 0 9 42).")
530 self._args.add_argument("--iov-id", default=None, type=int,
531 help="IoVid of the iov to be considered")
532 self._args.add_argument(
533 "--iov-pattern",
534 default=None,
535 help="whitespace-separated string with pattern of the iov to be considered. "
536 " Use * to mark the fields that should be ignored. Valid patterns are 0 0 -1 -1"
537 " (a very specific IoV), 0 * -1 -1 (any iov that starts in any run of exp 0 and ends exactly in exp -1, run -1)"
538 ", * * 3 45 (any Iov ending in exp 3, run 45, regardless from where it starts).")
539 self._args.add_argument("--run-range", nargs=4, default=None, type=int,
540 metavar=("FIRST_EXP", "FIRST_RUN", "FINAL_EXP", "FINAL_RUN"),
541 help="Can be four numbers to limit the run range to be considered"
542 " Only iovs overlapping, even partially, with this range will be considered.")
543 self._args.add_argument("--fully-contained", action="store_true",
544 help="If given together with ``--run_range`` limit the list of payloads "
545 "to the ones fully contained in the given run range")
546 if self.whichcommand == "copy":
547 self._args.add_argument("--set-run-range", action="store_true",
548 help="If given together with ``--run_range`` modify the interval of validity"
549 " of partially overlapping iovs to be fully contained in the given run range")
550 self.iovfilter.add_arguments("payloads")
551 self._args.add_argument("--revision", metavar='revision', type=int,
552 help="Specify the revision of the payload to be removed")
553 self._args.add_argument("--dry-run", help="Don't do anything, just print what would be done",
554 action="store_true", default=False)
555 if self.whichcommand == "copy":
556 self._args.add_argument("--replace", help="Modify the iovs in the output tag to avoid overlaps",
557 action="store_true", default=False)
558 self._args.add_argument("-j", type=int, default=10, dest="nprocess",
559 help="Number of concurrent threads to use.")
560
561 def get_iovs(self):
562 """Get the iovs already filtered"""
563 if not self.iovfilter.check_arguments():
564 B2ERROR("Issue with arguments")
565
566 all_iovs = self.db.get_all_iovs(
567 self._args.tag,
568 run_range=self._args.run_range,
569 fully_contained=self._args.fully_contained,
570 message=str(self.iovfilter))
571 self.num_all_iovs = len(all_iovs)
572 iovs_to_return = []
573 for iov in all_iovs:
574 if self._args.iov_id and iov.iov_id != self._args.iov_id:
575 continue
576 if not self.iovfilter.check(iov.name):
577 continue
578 if self._args.iov_pattern and not fnmatch("{} {} {} {}".format(*iov.iov),
579 "{} {} {} {}".format(*self._args.iov_pattern.split())):
580 continue
581 if self._args.revision and iov.revision != self._args.revision:
582 continue
583 iovs_to_return.append(iov)
584 return iovs_to_return
585
586 def modify_db(self, func, func_args, whichcommand=None):
587 """Modify the database using multithreading"""
588 if whichcommand is None:
589 whichcommand = self.whichcommand
590 try:
591 with ThreadPoolExecutor(max_workers=self._args.nprocess) as pool:
592 start = time.monotonic()
593 for iov_num, _ in enumerate(pool.map(func, func_args), 1):
594 eta = (time.monotonic() - start) / iov_num * (len(func_args) - iov_num)
595 B2INFO(f"{iov_num}/{len(func_args)} iovs {self.past_dict[whichcommand]}, ETA: {eta:.1f} seconds")
596 except RuntimeError:
597 B2ERROR(f"Not all iovs could be {self.past_dict[whichcommand]}. This could be a server/network problem "
598 "or the destination globaltag was not writeable.")
599 raise
600
601
602def command_iovs_delete(args, db=None):
603 """
604 Delete iovs from a globaltag
605
606 This command allows to delete the iovs from a globaltags, optionally limiting the iovs to be deleted to those
607 of a specific payload, revision, IoVid or run range.
608 """
609 command_iovs_helper = CommandIoVsHelper("delete", args, db)
610 if db is None:
611 command_iovs_helper.add_arguments()
612 return
613
614 iovs_to_delete = command_iovs_helper.get_iovs()
615
616 table = [[i.iov_id, i.name, i.revision] + list(i.iov) for i in iovs_to_delete]
617 table.insert(0, ["IovId", "Name", "Rev", "First Exp", "First Run", "Final Exp", "Final Run"])
618 columns = [9, "+", 6, 6, 6, 6, 6]
619
620 if command_iovs_helper.num_all_iovs == len(iovs_to_delete) and args.run_range is None:
621 B2WARNING(f"All the iovs in the globaltag {args.tag} will be deleted!")
622 gt_check = input('Please enter the global tag name again to confirm this action: ')
623 if gt_check != args.tag:
624 B2ERROR('global tag names do not match.')
625 return 1
626
627 B2INFO("Deleting the following iovs")
628 pretty_print_table(table, columns)
629
630 if not args.dry_run:
631 set_cdb_authentication_token(db, args.auth_token)
632 try:
633 command_iovs_helper.modify_db(db.delete_iov, [i.iov_id for i in iovs_to_delete])
634 except RuntimeError:
635 return 1
636
637 return 0
638
639
640def command_iovs_copy(args, db=None):
641 """
642 Copy iovs from a globaltag to another one
643
644 This command allows to copy the iovs from a globaltags to another one, optionally limiting
645 the iovs to be copied to those of a specific payload, revision, IoV id or run range.
646 """
647 command_iovs_helper = CommandIoVsHelper("copy", args, db)
648 if db is None:
649 command_iovs_helper.add_arguments()
650 return
651
652 iovs_to_copy = command_iovs_helper.get_iovs()
653
654 # make sure that there are no overlaps in the iovs to copy and set run range if needed
655 by_name = defaultdict(lambda: IoVSet(allow_overlaps=False))
656 for iov in iovs_to_copy:
657 if args.run_range and args.set_run_range:
658 iov.iov = (IntervalOfValidity(*iov.iov) & IntervalOfValidity(*args.run_range)).tuple
659 try:
660 by_name[iov.name].add(iov.iov)
661 except ValueError as e:
662 B2ERROR(f"Overlap for payload {iov.name} r{iov.revision}: {e}")
663 return 1
664 del by_name
665
666 # make sure output tag exists
667 output_id = db.get_globalTagInfo(args.output)
668 if output_id is None:
669 B2ERROR("Output globaltag doesn't exist. Please create it first with a proper description")
670 return False
671 output_id = output_id["globalTagId"]
672
673 # get iovs already present in output global tag
674 iovs_output = db.get_all_iovs(args.output)
675 iovs_output.sort(key=lambda i: i.iov[:2])
676 payloads_output = defaultdict(list)
677 for iov in iovs_output:
678 iov.newiov = IoVSet([IntervalOfValidity(*iov.iov)])
679 payloads_output[iov.name].append(iov)
680
681 # remove from iovs to copy those that are already present in the output global tag
682 iovs_in_output = [(i.checksum, i.iov) for i in iovs_output]
683 iovs_to_copy = [i for i in iovs_to_copy if (i.checksum, i.iov) not in iovs_in_output]
684 del iovs_in_output
685
686 iovs_not_to_copy = []
687
688 # readjust the iovs already present if they overlap
689 # with the ones to be copied
690 for iov_to_copy in iovs_to_copy:
691 iov_ = IntervalOfValidity(*iov_to_copy.iov)
692 still_to_copy = True
693 for iov_present in payloads_output[iov_to_copy.name]:
694 if iov_present.newiov & iov_:
695 if still_to_copy and iov_to_copy.checksum == iov_present.checksum:
696 iov_present.newiov.add(iov_, allow_overlaps=True)
697 iovs_not_to_copy.append(iov_to_copy)
698 still_to_copy = False
699 else:
700 iov_present.newiov -= iov_
701
702 iovs_to_delete = []
703 iovs_to_modify = []
704 iovs_to_create = []
705
706 # Fill lists of iovs to be deleted, modified or added
707 for iov_present in sum(payloads_output.values(), []):
708 # iov_present was replaced by copied iovs so it should be deleted
709 if len(iov_present.newiov) == 0:
710 iovs_to_delete.append(iov_present)
711 # iov_present was partially overlapping by copied iovs so it should be modified
712 elif len(iov_present.newiov) == 1:
713 if not list(iov_present.newiov)[0].tuple == iov_present.iov:
714 iovs_to_modify.append((iov_present, list(iov_present.newiov)[0].tuple))
715 # iov_present was split by copied iovs so its first iov should be modified and new iovs should be created
716 else:
717 newiovs = list(iov_present.newiov)
718 iovs_to_modify.append((iov_present, newiovs[0].tuple))
719 for newiov in newiovs[1:]:
720 iovs_to_create.append((iov_present, newiov.tuple))
721
722 # If I fix the overlaps do not copy iovs that can be obtained extending iovs already present
723 if args.replace:
724 iovs_to_copy = [i for i in iovs_to_copy if i not in iovs_not_to_copy]
725
726 # Print nice table
727 table = [[i.iov_id, i.name, i.revision] + list(i.iov) for i in iovs_to_copy]
728 table.insert(0, ["IovId", "Name", "Rev", "First Exp", "First Run", "Final Exp", "Final Run"])
729 columns = [9, "+", 6, 6, 6, 6, 6]
730 B2INFO(f"Copying the following iovs to {args.output}")
731 pretty_print_table(table, columns)
732
733 if (iovs_to_delete or iovs_to_modify or iovs_to_create) and not args.replace:
734 B2WARNING("Inserting the iovs will create overlaps,"
735 f" to avoid them the changes below should be implemented on {args.tag}")
736
737 if iovs_to_delete:
738 B2WARNING("iovs to be deleted")
739 # Sort by payload name and iov for nicer logs
740 iovs_to_delete.sort(key=lambda x: x.iov[:2])
741 iovs_to_delete.sort(key=lambda x: x.name)
742 # Print nice table
743 table = [[i.iov_id, i.name, i.revision] + list(i.iov) for i in iovs_to_delete]
744 table.insert(0, ["IovId", "Name", "Rev", "First Exp", "First Run", "Final Exp", "Final Run"])
745 columns = [9, "+", 6, 6, 6, 6, 6]
746 pretty_print_table(table, columns)
747 if iovs_to_modify:
748 B2WARNING("iovs to be modified")
749 # Sort by payload name and iov for nicer logs
750 iovs_to_modify.sort(key=lambda x: x[0].iov[:2])
751 iovs_to_modify.sort(key=lambda x: x[0].name)
752 # Print nice table
753 table = [[i[0].iov_id, i[0].name, i[0].revision] + list(i[0].iov) + list(i[1]) for i in iovs_to_modify]
754 table.insert(0, ["IovId", "Name", "Rev", "Old First Exp", "Old First Run", "Old Final Exp", "Old Final Run",
755 "New First Exp", "New First Run", "New Final Exp", "New Final Run"])
756 columns = [9, "+", 6, 6, 6, 6, 6, 6, 6, 6, 6]
757 pretty_print_table(table, columns)
758 if iovs_to_create:
759 B2WARNING("iovs to be created")
760 # Sort by payload name and iov for nicer logs
761 iovs_to_create.sort(key=lambda x: x[1][:2])
762 iovs_to_create.sort(key=lambda x: x[0].name)
763 # Print nice table
764 table = [[i[0].name, i[0].revision] + list(i[1]) for i in iovs_to_create]
765 table.insert(0, ["Name", "Rev", "First Exp", "First Run", "Final Exp", "Final Run"])
766 columns = ["+", 6, 6, 6, 6, 6]
767 pretty_print_table(table, columns)
768
769 if (iovs_to_delete or iovs_to_modify or iovs_to_create) and not args.replace:
770 B2WARNING("To apply them use the option --replace")
771 B2WARNING("If instead you want these overlaps to be present enter"
772 " the output global tag name again to confirm this action: ")
773 gt_check = input()
774 if gt_check != args.output:
775 B2ERROR("global tag names do not match.")
776 return 1
777
778 if not args.dry_run:
779 set_cdb_authentication_token(db, args.auth_token)
780 try:
781 command_iovs_helper.modify_db(lambda args: db.create_iov(output_id, *args),
782 [(i.payload_id, *i.iov) for i in iovs_to_copy])
783 if args.replace:
784 command_iovs_helper.modify_db(db.delete_iov, [i.iov_id for i in iovs_to_delete], "delete")
785 command_iovs_helper.modify_db(lambda args: db.modify_iov(*args),
786 [(i[0].iov_id, *i[1]) for i in iovs_to_modify], "modify")
787 command_iovs_helper.modify_db(lambda args: db.create_iov(output_id, *args),
788 [(i[0].payload_id, *i[1]) for i in iovs_to_create], "create")
789 except RuntimeError:
790 return 1
791
792 return 0
793
794
795def command_iovs_modify(args, db=None):
796 """
797 Modify iovs from a globaltag
798
799 This command allows to modify the iovs from a globaltags, optionally limiting the iovs to be modified to those
800 of a specific payload, revision, IoV id or run range.
801 """
802 command_iovs_helper = CommandIoVsHelper("modify", args, db)
803 if db is None:
804 command_iovs_helper.add_arguments()
805 return
806
807 new_iov = args.new_iov.split()
808 # Transform integer values of iov in int and keep '*' as strings. Raise error if something different is given.
809 for i in range(len(new_iov)):
810 try:
811 new_iov[i] = int(new_iov[i])
812 except ValueError:
813 if new_iov[i] != '*':
814 raise ValueError(f"Invalid IOV value: {new_iov[i]} should be an integer or '*'")
815
816 iovs_to_modify = []
817 for iov in command_iovs_helper.get_iovs():
818 new_iow_ = new_iov.copy()
819 for i in range(len(new_iow_)):
820 if new_iow_[i] == '*':
821 new_iow_[i] = iov.iov[i]
822 iovs_to_modify.append((iov, new_iow_))
823
824 table = [[i[0].iov_id, i[0].name, i[0].revision] + list(i[0].iov) + list(i[1]) for i in iovs_to_modify]
825 table.insert(0, ["IovId", "Name", "Rev", "Old First Exp", "Old First Run", "Old Final Exp", "Old Final Run",
826 "New First Exp", "New First Run", "New Final Exp", "New Final Run"])
827 columns = [9, "+", 6, 6, 6, 6, 6, 6, 6, 6, 6]
828
829 B2INFO("Changing the following iovs")
830 pretty_print_table(table, columns)
831
832 if not args.dry_run:
833 set_cdb_authentication_token(db, args.auth_token)
834 try:
835 command_iovs_helper.modify_db(lambda args: db.modify_iov(*args),
836 [(i[0].iov_id, *i[1]) for i in iovs_to_modify])
837 except RuntimeError:
838 return 1
839
840 return 0
db
conditions_db.ConditionsDB instance
dict past_dict
Dictionary with past participles.
num_all_iovs
number of iovs before payload and revision selection
str whichcommand
from whichcommand it is called (copy, delete or modify)