Belle II Software  release-05-01-25
local_metadata.py
1 #!/usr/bin/env python3
2 
3 """
4 Module containing all functionality necessary to mange local metadata and payload
5 information.
6 """
7 
8 import sqlite3
9 from . import PayloadInformation
10 
11 
13  """
14  Class to handle local sqlite dump of conditions database metadata
15 
16  This class can create and read sqlite dumps of the central database in a format
17  compatible with the local metadata provider in basf2.
18  """
19 
20  APPLICATION_ID = 0xb2cdb
21 
23  SCHEMA_VERSION = 1
24 
25  SCHEMA_SQL = """
26  -- first drop all tables we care about
27  DROP VIEW IF EXISTS iov_payloads;
28  DROP INDEX IF EXISTS iov_index;
29  DROP TABLE IF EXISTS iovs;
30  DROP TABLE IF EXISTS globaltags;
31  DROP VIEW IF EXISTS full_payloads;
32  DROP TABLE IF EXISTS payloads;
33  DROP TABLE IF EXISTS payloadNames;
34  DROP TABLE IF EXISTS baseUrls;
35 
36  -- table to store the base urls for normalization: Mostly we have the
37  -- same base url for all payloads but we only want to store the string
38  -- once so give it an id
39  CREATE TABLE baseUrls (
40  baseUrlId INTEGER PRIMARY KEY,
41  baseUrl text UNIQUE
42  );
43 
44  -- table to store all distinct payload names only once
45  CREATE TABLE payloadNames (
46  payloadNameId INTEGER PRIMARY KEY,
47  payloadName TEXT UNIQUE
48  );
49 
50  -- Payload information for each payload.
51  -- payloadId, revision, checksum, payloadUrl are taken directly without
52  -- change from the central database.
53  -- payloadName and baseUrl are put into local tables defined above,
54  -- these ids are local to the file.
55  -- payloadUrl is taken from the database except if it follows the usual
56  -- `dbstore/{payloadName}/dbstore_{payloadName}_rev_{revision}.root` in
57  -- which case it is set to NULL and created on the fly from the pattern.
58  CREATE TABLE payloads (
59  payloadId INTEGER PRIMARY KEY,
60  payloadNameId INTEGER NOT NULL,
61  revision INTEGER NOT NULL,
62  checksum text NOT NULL,
63  payloadUrl text DEFAULT NULL,
64  baseUrlId INTEGER NOT NULL,
65  CONSTRAINT name_rev UNIQUE (payloadNameId, revision),
66  FOREIGN KEY (payloadNameId) REFERENCES payloadNames (payloadNameId)
67  ON UPDATE CASCADE ON DELETE RESTRICT
68  FOREIGN KEY (baseUrlId) REFERENCES baseUrls (baseUrlId)
69  ON UPDATE CASCADE ON DELETE RESTRICT
70  );
71 
72  -- Payload information with the local ids and the payloadUrl resolved
73  -- Information in this view is identical to the central server
74  CREATE VIEW full_payloads AS
75  SELECT payloadId, payloadName, revision, checksum,
76  ifnull(payloadUrl, 'dbstore/' || payloadName || '/dbstore_' ||
77  payloadName || '_rev_' || revision || '.root')
78  as payloadUrl, baseUrl
79  FROM payloads NATURAL JOIN payloadNames NATURAL JOIN baseUrls;
80 
81  -- table for all globaltags in this file. All values taken directly from
82  -- the central server
83  CREATE TABLE globaltags (
84  globalTagId INTEGER PRIMARY KEY,
85  globalTagName text UNIQUE,
86  globalTagStatus text NOT NULL
87  );
88 
89  -- table for all iovs in all globaltags in this file, all values taken
90  -- directly from the central server
91  CREATE TABLE iovs (
92  globalTagId INTEGER NOT NULL,
93  payloadId INTEGER NOT NULL,
94  firstExp INTEGER NOT NULL,
95  firstRun INTEGER NOT NULL,
96  finalExp INTEGER NOT NULL,
97  finalRun INTEGER NOT NULL,
98  FOREIGN KEY (globalTagId) REFERENCES globaltags (globalTagId)
99  ON UPDATE CASCADE ON DELETE CASCADE,
100  FOREIGN KEY (payloadId) REFERENCES payloads (payloadId)
101  ON UPDATE CASCADE ON DELETE RESTRICT
102  );
103  -- composite index on the iovs to exclude duplicates and allow
104  -- performant lookup
105  CREATE UNIQUE INDEX iov_index on iovs (
106  globalTagId, firstExp, firstRun, finalExp, finalRun, payloadId
107  );
108 
109  -- full view returning the full information on all iovs in all globaltags
110  CREATE VIEW iov_payloads AS
111  SELECT globalTagName, payloadId, payloadName, revision, checksum,
112  firstExp, firstRun, finalExp, finalRun, payloadUrl, baseUrl
113  FROM globaltags NATURAL JOIN iovs NATURAL JOIN full_payloads;
114  """
115 
116  def __init__(self, filename, mode="read"):
117  """
118  Open an sqlite database and make sure that the schema exists in the
119  correct version or create it if ``mode=overwrite``
120 
121  Arguments:
122  filename (str): name of the database file
123  readonly (str): how to open the file. Can be one of ``read`` to open
124  the file readonly, ``append` to append new data to an existing file
125  and ``overwrite`` to recreate all tables and overwrite the contents.
126  """
127 
128 
129  self._cache = {}
130 
131  self._database = None
132 
133  # connect to the database file ...
134  if mode == "read":
135  self._database = sqlite3.connect(f"file:{filename}?mode=ro", uri=True)
136  elif mode in ["append", "overwrite"]:
137  self._database = sqlite3.connect(filename)
138  else:
139  raise RuntimeError("invalid mode: please supply one of 'read', 'append', 'overwrite'")
140 
141  if mode == "overwrite":
142  # drop and recreate all tables
143  self._database.executescript(self.SCHEMA_SQL)
144  # and set the application id/schema version correctly
145  self._database.execute(f"PRAGMA application_id = {self.APPLICATION_ID}")
146  self._database.execute(f"PRAGMA user_version = {self.SCHEMA_VERSION}")
147  self._database.commit()
148  else:
149  # make sure the application id/schema version is the same
150  application_id = self._database.execute("PRAGMA application_id").fetchone()[0]
151  if application_id != self.APPLICATION_ID:
152  raise RuntimeError("Not a b2conditionsdb database file")
153  schema_version = self._database.execute("PRAGMA user_version").fetchone()[0]
154  if schema_version != self.SCHEMA_VERSION:
155  raise RuntimeError("Cannot use sqlite file: different schema version, please recreate")
156 
157  def get_payload_count(self):
158  """Get the number of distinct payloads known to this file"""
159  cursor = self._database.execute("SELECT count(*) from full_payloads")
160  return cursor.fetchone()[0]
161 
162  def _resolve_id(self, name, entity):
163  """
164  Resolve the id for a named entity in the database file.
165 
166  Create new entities on demand and cache all known entities
167 
168  Parameters:
169  name (str): name to lookup
170  entity (str): type of the entity, currently ``baseUrl`` or ``payloadName``
171  """
172  # fill existing entries on first access
173  if entity not in self._cache:
174  self._cache[entity] = {row[0]: row[1] for row in self._database.execute(f"SELECT {entity}, {entity}Id from {entity}s")}
175  # and then check if we have this entry, otherwise make a new one
176  cache = self._cache[entity]
177  if name not in cache:
178  cache[name] = len(cache) + 1
179  self._database.execute(f"INSERT INTO {entity}s ({entity}Id, {entity}) VALUES (?,?)", (cache[name], name))
180  return cache[name]
181 
182  def add_globaltag(self, tag_id, name, state, iovs):
183  """
184  Add a globaltag to the database file. If the globaltag already exists in
185  the file its contents will be replaced.
186 
187  Parameters: tag_id (str): id of the globaltag in the central database
188  name (str): name of the globaltag in the central database state (str):
189  state of the globaltag in the central database iovs
190  (list(PayloadInformation)): all iovs valid for this globaltag
191  """
192  self._database.execute("INSERT OR REPLACE INTO globaltags VALUES (?,?,?)", (tag_id, name, state))
193  # remove existing iovs ... we want to append globaltags but we want all globaltags to be correct
194  self._database.execute("DELETE from iovs WHERE globalTagId=?", (tag_id,))
195 
196  all_payloads = {}
197  all_iovs = []
198  for p in iovs:
199  if p.payload_id not in all_payloads:
200  base_url = self._resolve_id(p.base_url, 'baseUrl')
201  name = self._resolve_id(p.name, 'payloadName')
202  url = None
203  if p.payload_url.lstrip('/') != f"dbstore/{p.name}/dbstore_{p.name}_rev_{p.revision}.root":
204  url = p.payload_url
205  all_payloads[p.payload_id] = (p.payload_id, name, p.revision, p.checksum, url, base_url)
206  all_iovs.append((tag_id, p.payload_id) + p.iov)
207 
208  self._database.executemany("INSERT OR REPLACE INTO payloads VALUES (?,?,?,?,?,?)", all_payloads.values())
209  self._database.executemany("INSERT INTO iovs VALUES (?,?,?,?,?,?)", all_iovs)
210  # make sure everything is comitted
211  self._database.commit()
212  self._database.execute("VACUUM")
213 
214  def get_globaltags(self):
215  """Return the list of globaltags stored in the file
216 
217  Returns:
218  a list of (id, name, state) tuples for all globaltags
219  """
220  return [row for row in self._database.execute("SELECT globalTagId, globalTagName, globalTagStatus FROM globaltags "
221  "ORDER by globalTagName")]
222 
223  def get_payloads(self):
224  """Get all payloads existing in this file
225 
226  Returns:
227  a sorted list of `PayloadInformation` objects for all payloads defined in this file with the iov set to None
228  """
229  payloads = sorted([PayloadInformation(*row) for row in
230  self._database.execute("SELECT payloadId, payloadName, revision, checksum, "
231  "payloadUrl, baseUrl from full_payloads")])
232  return payloads
233 
234  def get_all_iovs(self, globalTag, exp=None, run=None, message=None):
235  """Get all iovs for a given globaltag
236 
237  Parameters:
238  globalTag (str): name of the globaltag
239  exp (int): experiment number to check (or None to return all iovs)
240  run (int): run number to check (or None to return all iovs)
241  message (str): ignored, just for compatibility with `ConditionsDB.get_all_iovs`
242 
243  Returns:
244  a sorted list of `PayloadInformation` objects
245  """
246  params = {"globalTag": globalTag}
247  query = """\
248  SELECT
249  payloadId, payloadName, revision, checksum, payloadUrl, baseUrl,
250  firstExp, firstRun, finalExp, finalRun
251  FROM iov_payloads
252  WHERE globalTagName=:globalTag"""
253  if exp is not None:
254  params.update({"exp": exp, "run": run})
255  query += """ AND\
256  ((firstExp==:exp AND firstRun<=:run) OR firstExp<:exp) AND
257  (finalExp<0 OR (finalRun<0 AND finalExp>=:exp) OR (finalExp>:exp) OR (finalExp==:exp AND finalRun>=:run))"""
258  iovs = sorted([PayloadInformation(*row[:6], iov_id=None, iov=row[6:]) for row in
259  self._database.execute(query, params)])
260  return iovs
conditions_db.local_metadata.LocalMetadataProvider.APPLICATION_ID
int APPLICATION_ID
Application ID to be stored int the sqlite file.
Definition: local_metadata.py:20
conditions_db.local_metadata.LocalMetadataProvider.__init__
def __init__(self, filename, mode="read")
Definition: local_metadata.py:116
conditions_db.local_metadata.LocalMetadataProvider.get_all_iovs
def get_all_iovs(self, globalTag, exp=None, run=None, message=None)
Definition: local_metadata.py:234
conditions_db.local_metadata.LocalMetadataProvider.add_globaltag
def add_globaltag(self, tag_id, name, state, iovs)
Definition: local_metadata.py:182
conditions_db.local_metadata.LocalMetadataProvider
Definition: local_metadata.py:12
conditions_db.local_metadata.LocalMetadataProvider.SCHEMA_SQL
string SCHEMA_SQL
SQL script to create all necessary tables and views.
Definition: local_metadata.py:25
conditions_db.local_metadata.LocalMetadataProvider._database
_database
sqlite Database connection
Definition: local_metadata.py:131
conditions_db.local_metadata.LocalMetadataProvider._resolve_id
def _resolve_id(self, name, entity)
Definition: local_metadata.py:162
conditions_db.local_metadata.LocalMetadataProvider.get_payload_count
def get_payload_count(self)
Definition: local_metadata.py:157
conditions_db.local_metadata.LocalMetadataProvider.SCHEMA_VERSION
int SCHEMA_VERSION
Schema version, to be increased when the table definitions change so that we can check for safe appen...
Definition: local_metadata.py:23
conditions_db.local_metadata.LocalMetadataProvider.get_globaltags
def get_globaltags(self)
Definition: local_metadata.py:214
conditions_db.local_metadata.LocalMetadataProvider.get_payloads
def get_payloads(self)
Definition: local_metadata.py:223
conditions_db.local_metadata.LocalMetadataProvider._cache
_cache
Cache name->id mappings from the database.
Definition: local_metadata.py:129