#!/usr/bin/env python3
##########################################################################
# basf2 (Belle II Analysis Software Framework) #
# Author: The Belle II Collaboration #
# #
# See git log for contributors and copyright holders. #
# This file is licensed under LGPL-3.0, see LICENSE.md. #
##########################################################################
"""
conditions_db.iov
-----------------
This module contains classes to work with validity intervals. There's a class
for a single interval, `IntervalOfValidity` and a class to manage a set of
validities, `IoVSet`, which can be used to manipulate iov ranges
"""
import math
from itertools import product
[docs]
class IntervalOfValidity:
"""
Interval of validity class to support set operations like union and
intersection.
An interval of validity is a set of runs for which something is valid. An
IntervalOfValidity consists of a `first` valid run and a `final` valid run.
Warning:
The `final` run is inclusive so the the validity is including the final run.
Each run is identified by a experiment number and a run number. Accessing
`first` or `final` will return a tuple ``(experiment, run)`` but the
elements can also be accessed separately with `first_exp`, `first_exp`,
`final_exp` and `final_run`.
For `final` there's a special case where either the run or both, the run and
the experiment number are infinite. This means the validity extends to all
values. If only the run number is infinite then it's valid for all further
runs in this experiment. If both are infinite the validity extends to everything.
For simplicity ``-1`` can be passed in instead of infinity when creating objects.
"""
def __init__(self, *iov):
"""Create a new object.
It can be either instantiated by providing four values or one tuple/list
with four values for first_exp, first_run, final_exp, final_run
"""
if len(iov) == 1 and isinstance(iov[0], (list, tuple)):
iov = iov[0]
if len(iov) != 4:
raise ValueError("A iov should have four values")
#: tuple with the first valid exp, run
self.__first = tuple(iov[:2])
#: tuple with the final valid exp, run
self.__final = tuple(math.inf if x == -1 else x for x in iov[2:])
if math.isinf(self.__final[0]) and not math.isinf(self.__final[1]):
raise ValueError(f"Unlimited final experiment but not unlimited run: {self}")
if self.__first[0] > self.__final[0]:
raise ValueError(f"First exp larger than final exp: {self}")
if self.__first[0] == self.__final[0] and self.__first[1] > self.__final[1]:
raise ValueError(f"First run larger than final run: {self}")
if self.__first[0] < 0 or self.__first[1] < 0:
raise ValueError(f"Negative first exp or run: {self}")
[docs]
@staticmethod
def always():
"""Return an iov that is valid everywhere
>>> IntervalOfValidity.always()
(0, 0, inf, inf)
"""
return IntervalOfValidity(0, 0, -1, -1)
@property
def first(self):
"""Return the first valid experiment,run"""
return self.__first
@property
def first_exp(self):
"""Return the first valid experiment"""
return self.__first[0]
@property
def first_run(self):
"""Return the first valid run"""
return self.__first[1]
@property
def final(self):
"""Return the final valid experiment,run"""
return self.__final
@property
def final_exp(self):
"""Return the final valid experiment"""
return self.__final[0]
@property
def final_run(self):
"""Return the final valid run"""
return self.__final[1]
def __repr__(self):
"""Return a printable representation"""
return str(self.__first + self.__final)
def __eq__(self, other):
"""Check for equality"""
if not isinstance(other, IntervalOfValidity):
return NotImplemented
return (self.__first, self.__final) == (other.__first, other.__final)
def __lt__(self, other):
"""Sort by run values"""
if not isinstance(other, IntervalOfValidity):
return NotImplemented
return (self.__first, self.__final) < (other.__first, other.__final)
def __and__(self, other):
"""Intersection between iovs. Will return None if the payloads don't overlap"""
if not isinstance(other, IntervalOfValidity):
return NotImplemented
return self.intersect(other)
def __or__(self, other):
"""Union between iovs. Will return None if the iovs don't overlap or
connect to each other"""
if not isinstance(other, IntervalOfValidity):
return NotImplemented
return self.union(other, False)
def __sub__(self, other):
"""Difference between iovs. Will return None if nothing is left over"""
if not isinstance(other, IntervalOfValidity):
return NotImplemented
return self.subtract(other)
def __hash__(self):
"""Make object hashable"""
return hash((self.__first, self.__final))
[docs]
def subtract(self, other):
"""Return a new iov with the validity of the other removed.
Will return None if everything is removed.
Warning:
If the other iov is in the middle of the validity we will return a
tuple of two new iovs
>>> iov1 = IntervalOfValidity(0,0,10,-1)
>>> iov2 = IntervalOfValidity(5,0,5,-1)
>>> iov1 - iov2
((0, 0, 4, inf), (6, 0, 10, inf))
"""
if other.first <= self.first and other.final >= self.final:
# full overlap
return None
if other.first > self.first and other.final < self.final:
# the one we want to remove is in the middle, return a pair of iovs
# by subtracting two extended once
iov1 = self.subtract(IntervalOfValidity(other.first + (-1, -1)))
iov2 = self.subtract(IntervalOfValidity((0, 0) + other.final))
return (iov1, iov2)
if other.first <= self.final and other.final >= self.first:
# one sided overlap, figure out which side and calculate the remainder
if self.first < other.first:
end_run = other.first_run - 1
end_exp = other.first_exp if end_run >= 0 else other.first_exp - 1
return IntervalOfValidity(self.first + (end_exp, end_run))
else:
start_run = other.final_run + 1
start_exp = other.final_exp
if math.isinf(other.final_run):
start_exp += 1
start_run = 0
return IntervalOfValidity((start_exp, start_run) + self.final)
# no overlap so return unchanged
return self
[docs]
def intersect(self, other):
"""Intersection with another iov.
Will return None if the payloads don't overlap
>>> iov1 = IntervalOfValidity(1,0,2,5)
>>> iov2 = IntervalOfValidity(2,0,2,-1)
>>> iov3 = IntervalOfValidity(2,10,5,-1)
>>> iov1.intersect(iov2)
(2, 0, 2, 5)
>>> iov2.intersect(iov3)
(2, 10, 2, inf)
>>> iov3.intersect(iov1) is None
True
"""
if other.first <= self.final and other.final >= self.first:
return IntervalOfValidity(*(max(self.first, other.first) + min(self.final, other.final)))
return None
[docs]
def union(self, other, allow_startone=False):
"""
Return the union with another iov.
>>> iov1 = IntervalOfValidity(1,0,1,-1)
>>> iov2 = IntervalOfValidity(2,0,2,-1)
>>> iov3 = IntervalOfValidity(2,10,5,-1)
>>> iov1.union(iov2)
(1, 0, 2, inf)
>>> iov2.union(iov3)
(2, 0, 5, inf)
>>> iov3.union(iov1) is None
True
Warning:
This method will return None if the iovs don't overlap or connect to
each other as no union can be formed.
Parameters:
other (IntervalOfValidity): IoV to calculate the union with
allow_startone (bool): If True we will consider run 0 and run 1 the
first run in an experiment. This means that if one of the iovs has
un unlimited final run it can be joined with the other iov if the
experiment number increases and the iov starts at run 0 and 1. If
this is False just run 0 is considered the next run.
>>> iov1 = IntervalOfValidity(0,0,0,-1)
>>> iov2 = IntervalOfValidity(1,1,1,-1)
>>> iov1.union(iov2, False) is None
True
>>> iov1.union(iov2, True)
(0, 0, 1, inf)
"""
# check the trivial case of overlapping
if other.first <= self.final and other.final >= self.first:
return IntervalOfValidity(min(self.first, other.first) + max(self.final, other.final))
# ok, let's do the less simple case where they don't overlap but join directly
for i1, i2 in (self, other), (other, self):
if (i1.first == (i2.final_exp, i2.final_run + 1) or
(math.isinf(i2.final_run) and (i1.first_exp == i2.final_exp + 1) and
(i1.first_run == 0 or allow_startone and i1.first_run == 1))):
return IntervalOfValidity(i2.first + i1.final)
# no union possible: not directly connected and not overlapping
return None
[docs]
def contains(self, exp, run):
"""Check if a run is part of the validity"""
return self.first <= (exp, run) <= self.final
@property
def is_open(self):
"""Check whether the iov is valid until infinity"""
#: Doxygen complains without this string.
return self.final == (math.inf, math.inf)
@property
def tuple(self):
"""Return the iov as a tuple with experiment/run numbers replaced with -1
This is mostly helpful where infinity is not supported and is how the
intervals are represented in the database.
>>> a = IntervalOfValidity.always()
>>> a
(0, 0, inf, inf)
>>> a.tuple
(0, 0, -1, -1)
"""
return self.__first + tuple(-1 if math.isinf(x) else x for x in self.__final)
[docs]
class IoVSet:
"""A set of iovs.
This class allows to combine iovs into a set. New iovs can be added with
`add()` and will be combined with existing iovs if possible.
The final, minimal number of iovs can be obtained with the `iovs` property
>>> a = IoVSet()
>>> a.add((0,0,0,2))
>>> a.add((0,3,0,5))
>>> a.add((0,8,0,9))
>>> a
{(0, 0, 0, 5), (0, 8, 0, 9)}
"""
def __init__(self, iterable=None, *, allow_overlaps=False, allow_startone=False):
"""Create a new set.
>>> a = IoVSet([IntervalOfValidity(3,6,3,-1), (0,0,3,5)])
>>> a
{(0, 0, 3, inf)}
Parameters:
iterable: if not None it should be an iterable of IntervalOfValidity
objects or anything that can be converted to an IntervalOfValidity.
allow_overlaps (bool): If False adding which overlaps with any
existing iov in the set will raise a ValueError.
allow_startone (bool): If True also join iovs if one covers the
whole experiment and the next one starts at run 1 in the next
experiment. If False they will only be joined if the next one
starts at run 0.
"""
#: The set of iovs
self.__iovs = set()
#: Whether or not we raise an error on overlaps
self.__allow_overlaps = allow_overlaps
#: Whether or not run 1 will be also considered the first run when
# combining iovs between experiments
self.__allow_startone = allow_startone
if iterable is not None:
for element in iterable:
self.add(element)
[docs]
def add(self, iov, allow_overlaps=None):
"""
Add a new iov to the set.
The new iov be combined with existing iovs if possible. After the
operation the set will contain the minimal amount of separate iovs
possible to represent all added iovs
>>> a = IoVSet()
>>> a.add((0, 0, 0, 2))
>>> a.add((0, 3, 0, 5))
>>> a.add((0, 8, 0, 9))
>>> a
{(0, 0, 0, 5), (0, 8, 0, 9)}
>>> a.add(IoVSet([(10, 0, 10, 1), (10, 2, 10, -1)]))
>>> a
{(0, 0, 0, 5), (0, 8, 0, 9), (10, 0, 10, inf)}
Be aware, by default it's not possible to add overlapping iovs to the set.
This can be changed either on construction or per `add` call using
``allow_overlap``
>>> a.add((0, 2, 0, 3))
Traceback (most recent call last):
...
ValueError: Overlap between (0, 0, 0, 5) and (0, 2, 0, 3)
>>> a.add((0, 2, 0, 3), allow_overlaps=True)
>>> a
{(0, 0, 0, 5), (0, 8, 0, 9), (10, 0, 10, inf)}
Parameters:
iov (Union[IoVSet, IntervalOfValidity, tuple(int)]): IoV or
set of IoVs to add to this set
allow_overlaps (bool): Can be used to override global overlap setting
of this set to allow/restrict overlaps for a single insertion
operation
Warning:
This method modifies the set in place
"""
# check whether we override overlap settings
if allow_overlaps is None:
allow_overlaps = self.__allow_overlaps
# we can add a set to a set :D
if isinstance(iov, IoVSet):
for element in iov:
self.add(element, allow_overlaps)
return
# make sure it's actually an IoV, this will raise an error on failure
if not isinstance(iov, IntervalOfValidity):
iov = IntervalOfValidity(iov)
# and now check for all existing iovs ... (but use a copy since we modify the set)
for existing in list(self.__iovs):
# if there's an overlap to the new iov
if (not allow_overlaps) and (existing & iov):
raise ValueError(f"Overlap between {existing} and {iov}")
# and if they can be combined to a bigger iov
combined = existing.union(iov, self.__allow_startone)
# if we now have a combined iov, remove the one that we were able to
# combine it with from the existing iovs because we now check
# against the combined one. Since the only way to add a new iov is
# this loop we know all previous existing iovs we checked before
# didn't have a union with this new iov or any other existing iovs
# so if the just check the remaining iovs against the new combined
# one we can cascade combine all iovs in one go.
if combined is not None:
self.__iovs.remove(existing)
iov = combined
# done, we now have a new iov which combines all existing iovs it had an
# overlap with and we removed the existing iovs so nothing else to do
# but add the iov back in the list
self.__iovs.add(iov)
[docs]
def remove(self, iov):
"""Remove an iov or a set of iovs from this set
After this operation the set will not be valid for the given iov or set
of iovs:
>>> a = IoVSet()
>>> a.add((0,0,10,-1))
>>> a.remove((1,0,1,-1))
>>> a.remove((5,0,8,5))
>>> a
{(0, 0, 0, inf), (2, 0, 4, inf), (8, 6, 10, inf)}
>>> a.remove(IoVSet([(3,0,3,10), (3,11,3,-1)]))
>>> a
{(0, 0, 0, inf), (2, 0, 2, inf), (4, 0, 4, inf), (8, 6, 10, inf)}
Parameters:
iov (Union[IoVSet, IntervalOfValidity, tuple(int)]): IoV or
set of IoVs to remove from this set
Warning:
This method modifies the set in place
"""
# we can remove a set from a set :D
if isinstance(iov, IoVSet):
for element in iov:
self.remove(element)
return
# make sure it's actually an IoV, this will raise an error on failure
if not isinstance(iov, IntervalOfValidity):
iov = IntervalOfValidity(iov)
# and subtract the iov from all existing iovs
for existing in list(self.__iovs):
delta = existing - iov
if delta != existing:
self.__iovs.remove(existing)
if isinstance(delta, tuple):
# got two new iovs, apparently we split the old one
for new in delta:
self.__iovs.add(new)
elif delta is not None:
self.__iovs.add(delta)
[docs]
def intersect(self, iov):
"""Intersect this set with another set and return a new set
which is valid exactly where both sets have been valid before
>>> a = IoVSet()
>>> a.add((0,0,10,-1))
>>> a.intersect((5,0,20,-1))
{(5, 0, 10, inf)}
>>> a.intersect(IoVSet([(0,0,3,-1), (9,0,20,-1)]))
{(0, 0, 3, inf), (9, 0, 10, inf)}
Parameters:
iov (Union[IoVSet, IntervalOfValidity, tuple(int)]): IoV or
set of IoVs to intersect with this set
"""
if not isinstance(iov, (IoVSet, IntervalOfValidity)):
iov = IntervalOfValidity(iov)
if isinstance(iov, IntervalOfValidity):
iov = IoVSet([iov])
# ok for all combinations a,b from set1 and set2 check the intersection
# and if not empty add to the result
result = IoVSet()
for a, b in product(self.iovs, iov.iovs):
c = a & b
if c:
result.add(c)
return result
[docs]
def contains(self, iov):
"""
Check if an iov is fully covered by the set
>>> a = IoVSet([(0,0,2,-1), (5,0,5,-1)])
>>> a.contains((0,0,1,-1))
True
>>> a.contains(IntervalOfValidity(0,0,3,2))
False
>>> a.contains(IoVSet([(0,1,1,23), (5,0,5,23)]))
True
>>> a.contains(IoVSet([(0,1,1,23), (5,0,6,23)]))
False
>>> a.contains((3,0,4,-1))
False
Parameters:
iov (Union[IoVSet, IntervalOfValidity, tuple(int)]): IoV or
set of IoVs to be checked
Returns:
True if the full iov or all the iovs in the given set are fully
present in this set
"""
# check if the whole set is in this set: all iovs need to be in here
if isinstance(iov, IoVSet):
return all(e in self for e in iov)
# make sure it's actually an IoV, this will raise an error on failure
if not isinstance(iov, IntervalOfValidity):
iov = IntervalOfValidity(iov)
# and then check all iovs in the set if they cover it
for existing in self.__iovs:
if iov - existing is None:
return True
return False
[docs]
def overlaps(self, iov):
"""Check if the given iov overlaps with this set.
In contrast to `contains` this doesn't require the given iov to be fully
covered. It's enough if the any run covered by the iov is also covered
by this set.
>>> a = IoVSet([(0,0,2,-1), (5,0,5,-1)])
>>> a.overlaps((0,0,1,-1))
True
>>> a.overlaps(IntervalOfValidity(0,0,3,2))
True
>>> a.overlaps(IoVSet([(0,1,1,23), (5,0,5,23)]))
True
>>> a.overlaps(IoVSet([(0,1,1,23), (5,0,6,23)]))
True
>>> a.overlaps((3,0,4,-1))
False
Parameters:
iov (Union[IoVSet, IntervalOfValidity, tuple(int)]): IoV or
set of IoVs to be checked
Returns:
True if the iov or any of the iovs in the given set overlap with any
iov in this set
"""
if not isinstance(iov, (IoVSet, IntervalOfValidity)):
iov = IntervalOfValidity(iov)
if isinstance(iov, IntervalOfValidity):
iov = IoVSet([iov])
for a, b in product(self.iovs, iov.iovs):
c = a & b
if c:
return True
return False
[docs]
def copy(self):
"""Return a copy of this set"""
copy = IoVSet(allow_overlaps=self.__allow_overlaps, allow_startone=self.__allow_startone)
copy.__iovs = set(self.__iovs)
return copy
[docs]
def clear(self):
"""Clear all iovs from this set"""
self.__iovs = {}
@property
def iovs(self):
"""Return the set of valid iovs"""
return self.__iovs
@property
def first(self):
"""Return the first run covered by this iov set
>>> a = IoVSet([(3,0,3,10), (10,11,10,23), (0,0,2,-1), (5,0,5,-1)])
>>> a.first
(0, 0)
"""
if not self.__iovs:
return None
return min(self.iovs).first
@property
def final(self):
"""Return the final run covered by this iov set
>>> a = IoVSet([(3,0,3,10), (10,11,10,23), (0,0,2,-1), (5,0,5,-1)])
>>> a.final
(10, 23)
"""
if not self.__iovs:
return None
return max(self.iovs).final
@property
def gaps(self):
"""Return the gaps in the set. Any area not covered between the first
point of validity and the last
>>> a = IoVSet([(0,0,2,-1)])
>>> a.gaps
{}
>>> b = IoVSet([(0,0,2,-1), (5,0,5,-1)])
>>> b.gaps
{(3, 0, 4, inf)}
>>> c = IoVSet([(0,0,2,-1), (5,0,5,-1), (10,3,10,6)])
>>> c.gaps
{(3, 0, 4, inf), (6, 0, 10, 2)}
"""
if len(self.__iovs) < 2:
return IoVSet()
full_range = IoVSet([self.first + self.final])
return full_range - self
def __bool__(self):
"""Return True if the set is not empty
>>> a = IoVSet()
>>> a.add((0,0,1,-1))
>>> bool(a)
True
>>> a.clear()
>>> a
{}
>>> bool(a)
False
"""
return len(self.__iovs) > 0
def __contains__(self, iov):
"""Check if an iov is fully covered by the set"""
return self.contains(iov)
def __and__(self, other):
"""Return a new set that is the intersection between two sets
>>> a = IoVSet([(0,0,1,-1)])
>>> a & (1,0,2,-1)
{(1, 0, 1, inf)}
"""
return self.intersect(other)
def __or__(self, other):
"""
Return a new set that is the combination of two sets: The new set will
be valid everywhere any of the two sets were valid.
No check for overlaps will be performed but the result will inherit the
settings for further additions from the first set
>>> a = IoVSet([(0,0,1,-1)])
>>> a | (1,0,2,-1)
{(0, 0, 2, inf)}
>>> a | (3,0,3,-1)
{(0, 0, 1, inf), (3, 0, 3, inf)}
"""
copy = self.copy()
copy.add(other, allow_overlaps=True)
return copy
def __sub__(self, other):
"""
Return a new set which is only valid for where a is valid but not b.
See `remove` but this will not modify the set in place
>>> a = IoVSet([(0,0,-1,-1)])
>>> a - (1,0,2,-1)
{(0, 0, 0, inf), (3, 0, inf, inf)}
>>> a - (0,0,3,-1) - (10,0,-1,-1)
{(4, 0, 9, inf)}
>>> IoVSet([(0,0,1,-1)]) - (2,0,2,-1)
{(0, 0, 1, inf)}
"""
copy = self.copy()
copy.remove(other)
return copy
def __iter__(self):
"""Loop over the set of iovs"""
return iter(self.__iovs)
def __len__(self):
"""Return the number of validity intervals in this set"""
return len(self.__iovs)
def __repr__(self):
"""Return a printable representation"""
return '{' + ', '.join(str(e) for e in sorted(self.__iovs)) + '}'
