21 parser = argparse.ArgumentParser(description=
"Make a mapping file of file paths -> IoV")
22 parser.add_argument(
'--file_path_patterns', default=
'/hsm/belle2/bdata/Data/Raw/e0003/r*/**/*.root', type=str,
23 help=
'Lets take some file patterns. We could have put wildcards in more places but this is enough for testing.')
24 parser.add_argument(
'--output', default=
'file_iov_map.pkl', type=str,
25 help=
'Name of the output mapping file.')
30 help=
'Either take IoV from FileMetaData (option=metadata) or parse it from filepath (option=filepath).')
31 args = parser.parse_args()
33 file_path_patterns = [args.file_path_patterns, ]
34 print(file_path_patterns)
39 def from_raw_data_file_paths(file_path_patterns):
40 from caf.utils
import find_absolute_file_paths, parse_raw_data_iov
42 file_paths = find_absolute_file_paths(file_path_patterns)
45 for file_path
in file_paths:
46 file_to_iov[file_path] = parse_raw_data_iov(file_path)
52 def from_metadata_of_files(file_path_patterns):
54 from caf.utils
import make_file_to_iov_dictionary
56 def run_in_one_process():
58 Creates the file_to_iov dictionary but only one file at a time.
60 return make_file_to_iov_dictionary(file_path_patterns)
62 def run_with_multiprocessing(max_processes):
64 Creates the file_to_iov dictionary but using a Pool object to control the number of subprocesses.
65 Note that even though we're using a ThreadPool, we aren't bound by the GIL because we are subprocessing to
66 run b2file-metadata-show in each Thread.
68 from multiprocessing.pool
import ThreadPool
69 tp = ThreadPool(processes=max_processes)
70 mapping = make_file_to_iov_dictionary(file_path_patterns, polling_time=5, pool=tp)
76 return run_with_multiprocessing(max_processes=6)
79 if args.option ==
"metadata":
80 file_to_iov = from_metadata_of_files(file_path_patterns)
81 elif args.option ==
"filepath":
82 file_to_iov = from_raw_data_file_paths(file_path_patterns)
84 print(
"That wasn't one of the available options for this script. Run it again with no arguments to see the options.")
87 from pprint
import PrettyPrinter
88 pp = PrettyPrinter(indent=2)
89 pp.pprint(file_to_iov)
93 with open(args.output,
'bw')
as iov_map_file:
94 pickle.dump(file_to_iov, iov_map_file)
95 print(
"Saved dictionary to a file for later use.")