4 Module containing all functionality necessary to mange local metadata and payload
9 from .
import PayloadInformation
14 Class to handle local sqlite dump of conditions database metadata
16 This class can create and read sqlite dumps of the central database in a format
17 compatible with the local metadata provider in basf2.
20 APPLICATION_ID = 0xb2cdb
26 -- first drop all tables we care about
27 DROP VIEW IF EXISTS iov_payloads;
28 DROP INDEX IF EXISTS iov_index;
29 DROP TABLE IF EXISTS iovs;
30 DROP TABLE IF EXISTS globaltags;
31 DROP VIEW IF EXISTS full_payloads;
32 DROP TABLE IF EXISTS payloads;
33 DROP TABLE IF EXISTS payloadNames;
34 DROP TABLE IF EXISTS baseUrls;
36 -- table to store the base urls for normalization: Mostly we have the
37 -- same base url for all payloads but we only want to store the string
38 -- once so give it an id
39 CREATE TABLE baseUrls (
40 baseUrlId INTEGER PRIMARY KEY,
44 -- table to store all distinct payload names only once
45 CREATE TABLE payloadNames (
46 payloadNameId INTEGER PRIMARY KEY,
47 payloadName TEXT UNIQUE
50 -- Payload information for each payload.
51 -- payloadId, revision, checksum, payloadUrl are taken directly without
52 -- change from the central database.
53 -- payloadName and baseUrl are put into local tables defined above,
54 -- these ids are local to the file.
55 -- payloadUrl is taken from the database except if it follows the usual
56 -- `dbstore/{payloadName}/dbstore_{payloadName}_rev_{revision}.root` in
57 -- which case it is set to NULL and created on the fly from the pattern.
58 CREATE TABLE payloads (
59 payloadId INTEGER PRIMARY KEY,
60 payloadNameId INTEGER NOT NULL,
61 revision INTEGER NOT NULL,
62 checksum text NOT NULL,
63 payloadUrl text DEFAULT NULL,
64 baseUrlId INTEGER NOT NULL,
65 CONSTRAINT name_rev UNIQUE (payloadNameId, revision),
66 FOREIGN KEY (payloadNameId) REFERENCES payloadNames (payloadNameId)
67 ON UPDATE CASCADE ON DELETE RESTRICT
68 FOREIGN KEY (baseUrlId) REFERENCES baseUrls (baseUrlId)
69 ON UPDATE CASCADE ON DELETE RESTRICT
72 -- Payload information with the local ids and the payloadUrl resolved
73 -- Information in this view is identical to the central server
74 CREATE VIEW full_payloads AS
75 SELECT payloadId, payloadName, revision, checksum,
76 ifnull(payloadUrl, 'dbstore/' || payloadName || '/dbstore_' ||
77 payloadName || '_rev_' || revision || '.root')
78 as payloadUrl, baseUrl
79 FROM payloads NATURAL JOIN payloadNames NATURAL JOIN baseUrls;
81 -- table for all globaltags in this file. All values taken directly from
83 CREATE TABLE globaltags (
84 globalTagId INTEGER PRIMARY KEY,
85 globalTagName text UNIQUE,
86 globalTagStatus text NOT NULL
89 -- table for all iovs in all globaltags in this file, all values taken
90 -- directly from the central server
92 globalTagId INTEGER NOT NULL,
93 payloadId INTEGER NOT NULL,
94 firstExp INTEGER NOT NULL,
95 firstRun INTEGER NOT NULL,
96 finalExp INTEGER NOT NULL,
97 finalRun INTEGER NOT NULL,
98 FOREIGN KEY (globalTagId) REFERENCES globaltags (globalTagId)
99 ON UPDATE CASCADE ON DELETE CASCADE,
100 FOREIGN KEY (payloadId) REFERENCES payloads (payloadId)
101 ON UPDATE CASCADE ON DELETE RESTRICT
103 -- composite index on the iovs to exclude duplicates and allow
105 CREATE UNIQUE INDEX iov_index on iovs (
106 globalTagId, firstExp, firstRun, finalExp, finalRun, payloadId
109 -- full view returning the full information on all iovs in all globaltags
110 CREATE VIEW iov_payloads AS
111 SELECT globalTagName, payloadId, payloadName, revision, checksum,
112 firstExp, firstRun, finalExp, finalRun, payloadUrl, baseUrl
113 FROM globaltags NATURAL JOIN iovs NATURAL JOIN full_payloads;
118 Open an sqlite database and make sure that the schema exists in the
119 correct version or create it if ``mode=overwrite``
122 filename (str): name of the database file
123 readonly (str): how to open the file. Can be one of ``read`` to open
124 the file readonly, ``append` to append new data to an existing file
125 and ``overwrite`` to recreate all tables and overwrite the contents.
135 self.
_database = sqlite3.connect(f
"file:{filename}?mode=ro", uri=
True)
136 elif mode
in [
"append",
"overwrite"]:
137 self.
_database = sqlite3.connect(filename)
139 raise RuntimeError(
"invalid mode: please supply one of 'read', 'append', 'overwrite'")
141 if mode ==
"overwrite":
145 self.
_database.execute(f
"PRAGMA application_id = {self.APPLICATION_ID}")
146 self.
_database.execute(f
"PRAGMA user_version = {self.SCHEMA_VERSION}")
150 application_id = self.
_database.execute(
"PRAGMA application_id").fetchone()[0]
152 raise RuntimeError(
"Not a b2conditionsdb database file")
153 schema_version = self.
_database.execute(
"PRAGMA user_version").fetchone()[0]
155 raise RuntimeError(
"Cannot use sqlite file: different schema version, please recreate")
158 """Get the number of distinct payloads known to this file"""
159 cursor = self.
_database.execute(
"SELECT count(*) from full_payloads")
160 return cursor.fetchone()[0]
164 Resolve the id for a named entity in the database file.
166 Create new entities on demand and cache all known entities
169 name (str): name to lookup
170 entity (str): type of the entity, currently ``baseUrl`` or ``payloadName``
173 if entity
not in self.
_cache:
174 self.
_cache[entity] = {row[0]: row[1]
for row
in self.
_database.execute(f
"SELECT {entity}, {entity}Id from {entity}s")}
176 cache = self.
_cache[entity]
177 if name
not in cache:
178 cache[name] = len(cache) + 1
179 self.
_database.execute(f
"INSERT INTO {entity}s ({entity}Id, {entity}) VALUES (?,?)", (cache[name], name))
184 Add a globaltag to the database file. If the globaltag already exists in
185 the file its contents will be replaced.
187 Parameters: tag_id (str): id of the globaltag in the central database
188 name (str): name of the globaltag in the central database state (str):
189 state of the globaltag in the central database iovs
190 (list(PayloadInformation)): all iovs valid for this globaltag
192 self.
_database.execute(
"INSERT OR REPLACE INTO globaltags VALUES (?,?,?)", (tag_id, name, state))
194 self.
_database.execute(
"DELETE from iovs WHERE globalTagId=?", (tag_id,))
199 if p.payload_id
not in all_payloads:
203 if p.payload_url.lstrip(
'/') != f
"dbstore/{p.name}/dbstore_{p.name}_rev_{p.revision}.root":
205 all_payloads[p.payload_id] = (p.payload_id, name, p.revision, p.checksum, url, base_url)
206 all_iovs.append((tag_id, p.payload_id) + p.iov)
208 self.
_database.executemany(
"INSERT OR REPLACE INTO payloads VALUES (?,?,?,?,?,?)", all_payloads.values())
209 self.
_database.executemany(
"INSERT INTO iovs VALUES (?,?,?,?,?,?)", all_iovs)
215 """Return the list of globaltags stored in the file
218 a list of (id, name, state) tuples for all globaltags
220 return [row
for row
in self.
_database.execute(
"SELECT globalTagId, globalTagName, globalTagStatus FROM globaltags "
221 "ORDER by globalTagName")]
224 """Get all payloads existing in this file
227 a sorted list of `PayloadInformation` objects for all payloads defined in this file with the iov set to None
229 payloads = sorted([PayloadInformation(*row)
for row
in
230 self.
_database.execute(
"SELECT payloadId, payloadName, revision, checksum, "
231 "payloadUrl, baseUrl from full_payloads")])
235 """Get all iovs for a given globaltag
238 globalTag (str): name of the globaltag
239 exp (int): experiment number to check (or None to return all iovs)
240 run (int): run number to check (or None to return all iovs)
241 message (str): ignored, just for compatibility with `ConditionsDB.get_all_iovs`
244 a sorted list of `PayloadInformation` objects
246 params = {
"globalTag": globalTag}
249 payloadId, payloadName, revision, checksum, payloadUrl, baseUrl,
250 firstExp, firstRun, finalExp, finalRun
252 WHERE globalTagName=:globalTag"""
254 params.update({
"exp": exp,
"run": run})
256 ((firstExp==:exp AND firstRun<=:run) OR firstExp<:exp) AND
257 (finalExp<0 OR (finalRun<0 AND finalExp>=:exp) OR (finalExp>:exp) OR (finalExp==:exp AND finalRun>=:run))"""
258 iovs = sorted([PayloadInformation(*row[:6], iov_id=
None, iov=row[6:])
for row
in