Belle II Software  release-05-01-25
setup-01-preparelocaldb.py
1 #!/usr/bin/env python3
2 
3 """
4 Script to prepare a local database for HLT validation.
5 
6 This is more complicated as we would like because performance of HLT depends on
7 the payloads matching to the given run but we also want to valdiate new payloads
8 and/or new software versions. So as a "best guess" what could work we take with
9 the following priority
10 
11 1. any payloads in online that are unlimited.
12 2. any payloads in online valid for the validation run but no longer unlimited
13  *if* a payload with the same name still exists as unlimited in the online tag.
14 3. any payloads in staging_online
15 
16 We take all of these and put them in a local database file and download all the
17 necessary payload files.
18 """
19 
20 import os
21 import copy
22 from pathlib import Path
23 import multiprocessing
24 import functools
25 from conditions_db import ConditionsDB
26 from conditions_db.iov import IntervalOfValidity
27 from conditions_db.local_metadata import LocalMetadataProvider
28 from conditions_db.cli_download import download_payload
29 
30 
31 def update_payload(existing, payload, priority, source):
32  # normally we'd just take one but right now there are overlaps in
33  # online so better safe than sorry: take highest revision
34  update = payload.name not in existing
35  if not update:
36  other = existing[payload.name]
37  update = other.priority < priority or \
38  (other.priority == priority and other.revision < payload.revision)
39 
40  if update:
41  # keep for all runs
42  p = copy.copy(payload)
43  p.iov = (0, 0, -1, -1)
44  # and remember priority gt for debugging
45  p.priority = priority
46  p.source = source
47  existing[p.name] = p
48 
49 
50 def prepare_globaltag(db, exp, run):
51  payloads = {}
52 
53  # get the unlimited iovs from online ... and the ones for our particular run
54  base = db.get_all_iovs("online")
55  existing_names = set()
56  for p in base:
57  iov = IntervalOfValidity(p.iov)
58  if iov.is_open:
59  update_payload(payloads, p, 1, "online")
60  existing_names.add(p.name)
61 
62  for p in base:
63  iov = IntervalOfValidity(p.iov)
64  if not iov.is_open and iov.contains(exp, run) and p.name in existing_names:
65  update_payload(payloads, p, 2, f"e{exp}r{run}")
66 
67  # and take everything from staging
68  staging = db.get_all_iovs("staging_online")
69  for p in staging:
70  update_payload(payloads, p, 3, "staging")
71 
72  return sorted(payloads.values())
73 
74 
75 if __name__ == "__main__":
76  output_dir = Path("cdb")
77  metadata = output_dir / "metadata.sqlite"
78  if metadata.exists():
79  metadata.unlink()
80  else:
81  output_dir.mkdir(parents=True, exist_ok=True)
82 
83  db = ConditionsDB()
84 
85  # check if we have a prepared globaltag to check Also allow overriding this in
86  # a bamboo build variable if necessary. If we have a prepared globaltag we don't
87  # build our own version but just download it.
88  existing_gt = os.environ.get("VALIDATION_GLOBALTAG", "").strip()
89  if existing_gt:
90  payloads = db.get_all_iovs(existing_gt)
91  for p in payloads:
92  p.source = existing_gt
93  else:
94  payloads = prepare_globaltag(db, int(os.environ['VALIDATION_EXP']), int(os.environ['VALIDATION_RUN']))
95 
96  # Save under the name "online" in the local database file
97  localdb = LocalMetadataProvider(str(metadata), mode="overwrite")
98  localdb.add_globaltag(1, "online", "RUNNING", payloads)
99 
100  # Download all the payloads
101  downloader = downloader = functools.partial(download_payload, db, directory=output_dir)
102  with multiprocessing.Pool() as p:
103  p.map(downloader, payloads)
104 
105  # and print the final result
106  maxlen = max(len(p.name) for p in payloads)
107  print("Payloads taken: ")
108  for p in payloads:
109  print(f"{p.name:<{maxlen}s} {p.revision:7d} from {p.source}")
conditions_db.local_metadata
Definition: local_metadata.py:1
conditions_db.cli_download
Definition: cli_download.py:1
conditions_db.iov
Definition: iov.py:1