Belle II Software development
validationserver.py
1
9from typing import Dict, Any, List, Tuple
10from glob import glob
11import json
12import functools
13import time
14import datetime
15from multiprocessing import Process, Queue
16import os.path
17import argparse
18import logging
19import sys
20import queue
21import webbrowser
22import re
23import collections
24import configparser
25import requests
26
27# 3rd
28import cherrypy
29import gitlab
30
31# ours
32import json_objects
33from validationplots import create_plots
34import validationfunctions
35import validationpath
36
37g_plottingProcesses: Dict[str, Tuple[Process, Queue, Dict[str, Any]]] = ({})
38
39
40def get_revision_label_from_json_filename(json_filename: str) -> str:
41 """
42 Gets the label of a revision from the path to the revision.json file
43 for example results/r121/revision.json
44 will result in the label r121
45 This is useful if the results folder has been moved by the user
46 """
47 folder_part = os.path.split(json_filename)[0]
48 last_folder = os.path.basename(folder_part)
49
50 return last_folder
51
52
53def get_json_object_list(
54 results_folder: str, json_file_name: str
55) -> List[str]:
56 """
57 Searches one folder's sub-folder for json files of a
58 specific name and returns a combined list of the
59 json file's content
60 """
61
62 search_string = results_folder + "/*/" + json_file_name
63
64 found_revs = glob(search_string)
65 found_rev_labels = []
66
67 for r_file in found_revs:
68 # try loading json file
69 with open(r_file) as json_file:
70 data = json.load(json_file) # noqa
71
72 # always use the folder name as label
73 found_rev_labels.append(
74 get_revision_label_from_json_filename(r_file)
75 )
76
77 return found_rev_labels
78
79
80def deliver_json(file_name: str):
81 """
82 Simply load & parse a json file and return the
83 python objects
84 """
85
86 with open(file_name) as json_file:
87 data = json.load(json_file)
88 return data
89
90
91def create_revision_key(revision_names: List[str]) -> str:
92 """
93 Create a string key out of a revision list, which is handed to tho browser
94 in form of a progress key
95 """
96 return functools.reduce(lambda x, y: x + "-" + y, revision_names, "")
97
98
99def check_plotting_status(progress_key: str):
100 """
101 Check the plotting status via the supplied progress_key
102 """
103
104 if progress_key not in g_plottingProcesses:
105 return None
106
107 process, qu, last_status = g_plottingProcesses[progress_key]
108
109 # read latest message
110 try:
111 # read as much entries from the queue as possible
112 while not qu.empty():
113 msg = qu.get_nowait()
114 last_status = msg
115
116 # update the last status
117 g_plottingProcesses[progress_key] = (process, qu, last_status)
118 except queue.Empty:
119 pass
120
121 return last_status
122
123
124# todo: remove this, once we're certain that the bug was fixed!
125def warn_wrong_directory():
126 if not os.getcwd().endswith("html"):
127 print(
128 f"ERROR: Expected to be in HTML directory, but my current "
129 f"working directory is {os.getcwd()}; abspath: {os.getcwd()}."
130 )
131
132
133# todo: limit the number of running plotting requests & terminate hanging ones
134def start_plotting_request(
135 revision_names: List[str], results_folder: str
136) -> str:
137 """
138 Start a new comparison between the supplied revisions
139
140 Returns:
141 revision key
142 """
143
144 rev_key = create_revision_key(revision_names)
145
146 # still running a plotting for this combination ?
147 if rev_key in g_plottingProcesses:
148 logging.info(f"Plotting request for {rev_key} still running")
149 return rev_key
150
151 # create queue to stream progress, only one directional from parent to
152 # child
153 qu = Queue()
154
155 # start a new process for creating the plots
156 p = Process(
157 target=create_plots,
158 args=(
159 revision_names,
160 False,
161 qu,
162 # go one folder up, because this function
163 # expects the work dir, which contains
164 # the results folder
165 os.path.dirname(results_folder),
166 ),
167 )
168 p.start()
169 g_plottingProcesses[rev_key] = (p, qu, None)
170
171 logging.info(f"Started process for plotting request {rev_key}")
172
173 return rev_key
174
175
176"""
177Gitlab Integration
178
179Under here are functions that enable the validationserver to
180interact directly with the Gitlab project page to track and
181update issues.
182
183Requirement:
184Config file with project information and access token in the local
185machine. This is expected to be in the validation/config folder, in
186the same root directory as the html_static files.
187
188A check is performed to see if the config file exists with all the
189relevant details and all of Gitlab functionalities are enabled/disabled
190accordingly.
191
192When the server is being set up, a Gitlab object is created, which
193will be used subsequently to make all the API calls.
194
195As a final server initialization step, the project is queried to
196check if any of the current results are linked to existing issues
197and the result files are updated accordingly.
198
199The create/update issue functionality is accessible from the plot
200container. All the relevant pages are part of the
201validationserver cherry object.
202
203Issues created by the validation server will contain a block of
204automated code at the end of description and can be easily
205filtered from the GitLab issues page using the search string
206"Automated code, please do not delete". Relevant plots will be
207listed as a note in the issue page, along with the revision label.
208
209Function to upload files to Gitlab helps with pushing error plots
210to the project.
211"""
212
213
214def get_gitlab_config(
215 config_path: str
216) -> configparser.ConfigParser:
217 """
218 Parse the configuration file to be used to authenticate
219 GitLab API and retrieve relevant project info.
220
221 Returns:
222 gitlab configparser object
223 """
224
225 gitlab_config = configparser.ConfigParser()
226 gitlab_config.read(config_path)
227
228 return gitlab_config
229
230
231def create_gitlab_object(config_path: str) -> gitlab.Gitlab:
232 """
233 Establish connection with Gitlab using a private access key and return
234 a Gitlab object that can be used to make API calls. Default config
235 from the passed ini file will be used.
236
237 Returns:
238 gitlab object
239 """
240
241 gitlab_object = gitlab.Gitlab.from_config(
242 config_files=[config_path]
243 )
244 try:
245 gitlab_object.auth()
246 logging.info("Established connection with Gitlab")
247 except gitlab.exceptions.GitlabAuthenticationError:
248 gitlab_object = None
249 logging.warning(
250 "Issue with authenticating GitLab. "
251 "Please ensure access token is correct and valid. "
252 "GitLab Integration will be disabled."
253 )
254 except requests.exceptions.Timeout:
255 gitlab_object = None
256 logging.warning(
257 "GitLab servers feeling under the weather, DESY outage? "
258 "GitLab Integration will be disabled."
259 )
260
261 return gitlab_object
262
263
264def get_project_object(
265 gitlab_object: gitlab.Gitlab, project_id: str
266) -> 'gitlab.project':
267 """
268 Fetch Gitlab project associated with the project ID.
269
270 Returns:
271 gitlab project object
272 """
273
274 project = gitlab_object.projects.get(project_id, lazy=True)
275
276 return project
277
278
279def search_project_issues(
280 gitlab_object: gitlab.Gitlab,
281 search_term: str,
282 state: str = 'opened',
283) -> 'list[gitlab.issues]':
284 """
285 Search in the Gitlab for open issues that contain the
286 key phrase.
287
288 Returns:
289 gitlab project issues
290 """
291
292 issues = gitlab_object.issues.list(
293 search=search_term,
294 state=state,
295 lazy=True,
296 scope='all',
297 get_all=True,
298 )
299
300 return issues
301
302
303def update_linked_issues(
304 gitlab_object: gitlab.Gitlab, cwd_folder: str
305) -> None:
306 """
307 Fetch linked issues and update the comparison json files.
308
309 Returns:
310 None
311 """
312
313 # collect list of issues validation server has worked with
314 search_key = "Automated code, please do not delete"
315 issues = search_project_issues(gitlab_object, search_key)
316 past_issues = search_project_issues(gitlab_object, search_key, 'closed')
317
318 # find out the plots/scripts linked to the issues
319 # store closed issue ids as -ve numbers to distinguish them from open ones
320 plot_issues = collections.defaultdict(list)
321 script_issues = collections.defaultdict(list)
322 pattern = r"Relevant ([a-z]+): (\w+.*\w*)"
323 for i, issue in enumerate(issues+past_issues):
324 match = re.search(pattern, issue.description)
325 if match:
326 if match.groups()[0] == 'plot':
327 if i >= len(issues):
328 plot_issues[match.groups()[1]].append(-issue.iid)
329 else:
330 plot_issues[match.groups()[1]].append(issue.iid)
331 else:
332 script_issues[match.groups()[1]].append(issue.iid)
333
334 # get list of available revision hashes
335 rev_list = get_json_object_list(
337 validationpath.file_name_comparison_json,
338 )
339
340 for r in rev_list:
341 comparison_json_path = os.path.join(
343 r,
344 validationpath.file_name_comparison_json,
345 )
346 comparison_json = deliver_json(comparison_json_path)
347 for package in comparison_json["packages"]:
348 for plotfile in package.get("plotfiles"):
349 for plot in plotfile.get("plots"):
350 if plot["png_filename"] in plot_issues.keys():
351 plot["issue"] = plot_issues[plot["png_filename"]]
352 else:
353 plot["issue"] = []
354
355 with open(comparison_json_path, "w") as jsonFile:
356 json.dump(comparison_json, jsonFile, indent=4)
357
358 # get list of available revision labels
359 rev_list = get_json_object_list(
361 validationpath.file_name_results_json,
362 )
363 for r in rev_list:
364 revision_json_path = os.path.join(
366 r,
367 validationpath.file_name_results_json,
368 )
369 revision_json = deliver_json(revision_json_path)
370 for package in revision_json["packages"]:
371 for scriptfile in package.get("scriptfiles"):
372 if scriptfile["name"] in script_issues.keys():
373 scriptfile["issues"] = script_issues[scriptfile['name']]
374 else:
375 scriptfile["issues"] = []
376
377 with open(revision_json_path, "w") as jsonFile:
378 json.dump(revision_json, jsonFile, indent=4)
379
380
381def upload_file_gitlab(
382 file_path: str, project: 'gitlab.project'
383) -> Dict[str, str]:
384 """
385 Upload the passed file to the Gitlab project.
386
387 Returns:
388 uploaded gitlab project file object
389 """
390
391 uploaded_file = project.upload(
392 file_path.split("/")[-1], filepath=file_path
393 )
394
395 return uploaded_file
396
397
398def get_librarians(package: str) -> List[str]:
399 """
400 Function to get package librarian(s)' GitLab usernames. Temp solution
401 until the .librarians file directly provides Gitlab usernames.
402
403 Return:
404 list of librarians' Gitlab usernames
405 """
406
407 usernames = []
408 librarian_file = os.path.join(
410 package,
411 '.librarians'
412 )
413 try:
414 with open(librarian_file, 'r') as f:
415 librarians = f.readlines()
416 except FileNotFoundError:
417 logging.exception(
418 f"{librarian_file} couldn't be found. Corrupted package/librarian file?"
419 )
420 return usernames
421 # Temp workaround to fetch DESY -> Gitlab map
422 import importlib
423 desy_map_path = os.path.join(
424 "/home/b2soft/gitlab",
425 "account_map.py"
426 )
427 spec = importlib.util.spec_from_file_location('account_map', desy_map_path)
428 desy_map = importlib.util.module_from_spec(spec)
429 try:
430 spec.loader.exec_module(desy_map)
431 except FileNotFoundError:
432 logging.exception(
433 f"{desy_map_path} couldn't be found. Have you setup Gitlab Webhook?"
434 )
435 return usernames
436
437 for librarian in librarians:
438 usernames.append(desy_map.get_gitlab_account(librarian.rstrip()))
439
440 return usernames
441
442
443def parse_contact(
444 contact: str,
445 map_file: str,
446 package: str,
447 gitlab_object: gitlab.Gitlab
448) -> List[str]:
449 """
450 Parse string to find email id(s) and then match them with their Gitlab ids
451 using the userid map.
452
453 Returns :
454 Dictionary with list of Gitlab IDs and corresponding list of
455 Gitlab usernames.
456 """
457
458 email_regex = re.compile(r"([-!#-'*+/-9=?A-Z^-~]+(\.[-!#-'*+/-9=?A-Z^-~]+)*"
459 r"|\"([]!#-[^-~ \t]|(\\[\t -~]))+\")@([-!#-'*+/-9="
460 r"?A-Z^-~]+(\.[-!#-'*+/-9=?A-Z^-~]+)*|\[[\t -Z^-~]"
461 r"*])"
462 )
463 email_ids = re.finditer(email_regex, contact)
464 assignees = {
465 'gitlab_ids': [],
466 'usernames': [],
467 }
468
469 try:
470 with open(map_file, 'r') as f:
471 id_map = f.readlines()
472 except FileNotFoundError:
473 logging.exception(
474 f"{map_file} couldn't be found. Did you get the location right?"
475 )
476 email_ids = []
477
478 for email in email_ids:
479 try:
480 match = next(
481 (line for line in id_map if email.group() in line), None
482 )
483 if not match:
484 logging.error(
485 f"No userid found for {email} in the map, could it be that "
486 "they are (sadly) no longer in the collaboration?"
487 )
488 continue
489 username = match.split(' ')[1].rstrip()
490 assignees['usernames'].append(username)
491 except IndexError:
492 logging.error(
493 f"Map info {match} does not match the required format for map "
494 "'email gitlab_username'."
495 )
496 continue
497
498 # Assign to librarian(s) if no contact found
499 if not assignees['usernames']:
500 assignees['usernames'] = get_librarians(package)
501 logging.info(
502 "Couldn't find contact/id so assigning issue to the"
503 " package librarians."
504 )
505
506 for user in assignees['usernames']:
507 try:
508 assignees['gitlab_ids'].append(
509 gitlab_object.users.list(username=user)[0].id
510 )
511 except IndexError:
512 logging.error(
513 f"Could not find {user} in Gitlab."
514 )
515 continue
516 logging.info(
517 "Issue will be assigned to "
518 f"{[gitlab_object.users.get(id) for id in assignees['gitlab_ids']]}."
519 )
520
521 # to-do: add comment/note if no ids matched
522
523 return assignees
524
525
526def create_gitlab_issue(
527 title: str,
528 description: str,
529 uploaded_file: Dict[str, str],
530 assignees: Dict[str, List],
531 package: str,
532 project: 'gitlab.project'
533) -> str:
534 """
535 Create a new project issue with the passed title, description and package,
536 using Gitlab API.
537
538 Returns:
539 created issue id
540 """
541
542 issue = project.issues.create({"title": title,
543 "description": description,
544 "labels": [package, 'validation_issue']})
545
546 issue_note = issue.notes.create(
547 {"body": f'View the [error plot/log file]({uploaded_file["url"]}).'}
548 )
549
550 issue.assignee_ids = assignees['gitlab_ids']
551
552 # Workaround for Gitlab not allowing multiple assignees
553 if len(assignees['gitlab_ids']) > 1:
554 related_users = [f'@{user} ' for user in assignees['usernames'][1:]]
555 issue_note.body += f"\n\nPinging {' '.join(related_users)}"
556 issue_note.save()
557
558 issue.save()
559
560 logging.info(f"Created a new Gitlab issue - {issue.iid}")
561
562 return issue.iid
563
564
565def update_gitlab_issue(
566 issue_iid: str,
567 uploaded_file: Dict[str, str],
568 project: 'gitlab.project',
569 file_path: str,
570 rev_label: str
571) -> None:
572 """
573 Update an existing project issue with the passed plotfile.
574
575 Returns:
576 None
577 """
578
579 issue = project.issues.get(issue_iid)
580 name = file_path.split("/")[-1].split(".")[0]
581 package = file_path.split("/")[-2]
582 # check if this is a plot/script based on file format
583 issue_type = 'plot'
584 if 'log' == file_path.split("/")[-1].split(".")[-1]:
585 issue_type = 'script'
586 issue.notes.create(
587 {
588 "body": f'Related observation in validation of `{package}` package, `{name}`' +
589 f'{issue_type} in `{rev_label}` build. View the [error plot/log file]({uploaded_file["url"]}).'
590 }
591 )
592
593 issue.save()
594
595 logging.info(f"Updated existing Gitlab issue {issue.iid}")
596
597
598def update_scriptfile_issues_json(
599 revision_json_path: str,
600 scritptfile_name: str,
601 scritptfile_package: str,
602 issue_id: str
603) -> None:
604 """
605 Update the scriptfile's linked issues key in the relevant revision's
606 json file.
607
608 Returns:
609 None
610 """
611
612 revision_json = deliver_json(revision_json_path)
613 for package in revision_json["packages"]:
614 if package["name"] == scritptfile_package:
615 for scriptfile in package.get("scriptfiles"):
616 if (scriptfile["name"] == scritptfile_name):
617 scriptfile["issues"].append(issue_id)
618 break
619
620 with open(revision_json_path, "w") as jsonFile:
621 json.dump(revision_json, jsonFile, indent=4)
622
623
624def update_plot_issues_json(
625 comparison_json_path: str,
626 plot_name: str,
627 plot_package: str,
628 issue_id: str
629) -> None:
630 """
631 Update the plotfile's linked issues key in the relevant comparison
632 json file.
633
634 Returns:
635 None
636 """
637
638 comparison_json = deliver_json(comparison_json_path)
639 for package in comparison_json["packages"]:
640 if package["name"] == plot_package:
641 for plotfile in package.get("plotfiles"):
642 for plot in plotfile.get("plots"):
643 if (plot["png_filename"] == plot_name):
644 plot["issue"].append(issue_id)
645 break
646
647 with open(comparison_json_path, "w") as jsonFile:
648 json.dump(comparison_json, jsonFile, indent=4)
649
650
652
653 """
654 Root Validation class to handle non-static HTTP requests into the
655 validation server. The two main functions are to hand out compiled json
656 objects of revisions and comparisons and to start and monitor the
657 creation of comparison plots.
658
659 """
660
661 def __init__(self, working_folder, gitlab_object, gitlab_config, gitlab_map):
662 """
663 class initializer, which takes the path to the folders containing the
664 validation run results and plots (aka comparison), gitlab object and
665 config
666 """
667
668
669 self.working_folder = working_folder
670
671
672 self.last_restart = datetime.datetime.now()
673
674
676 os.environ["BELLE2_LOCAL_DIR"]
677 )
678
679
680 self.gitlab_object = gitlab_object
681
682
683 self.gitlab_config = gitlab_config
684
685
686 self.gitlab_map = gitlab_map
687
688
689 self.file_path = None
690
691 self.revision_label = None
692
693 self.contact = None
694
695 @cherrypy.expose
696 @cherrypy.tools.json_in()
697 @cherrypy.tools.json_out()
699 """
700 Triggers the start of a now comparison between the revisions supplied
701 in revision_list
702 """
703 rev_list = cherrypy.request.json["revision_list"]
704 logging.debug("Creating plots for revisions: " + str(rev_list))
705 progress_key = start_plotting_request(
706 rev_list,
708 )
709 return {"progress_key": progress_key}
710
711 @cherrypy.expose
712 def index(self):
713 """
714 forward to the static landing page if
715 the default url is used (like http://localhost:8080/)
716 """
717 raise cherrypy.HTTPRedirect("/static/validation.html")
718
719 @cherrypy.expose
720 def plots(self, *args):
721 """
722 Serve file from the html/plot directory.
723 :param args: For the request /plots/a/b/c, these will be the strings
724 "a", "b", "c"
725 """
726
727 warn_wrong_directory()
728
729 if len(args) < 3:
730 raise cherrypy.HTTPError(404)
731
732 tag_folder = os.path.relpath(
734 self.working_folder, args[:-2]
735 ),
737 )
738 path = os.path.join(tag_folder, *args[-2:])
739 return cherrypy.lib.static.serve_file(path)
740
741 @cherrypy.expose
742 @cherrypy.tools.json_in()
743 @cherrypy.tools.json_out()
745 """
746 Checks on the status of a comparison creation
747 """
748 progress_key = cherrypy.request.json["input"]
749 logging.debug("Checking status for plot creation: " + str(progress_key))
750 status = check_plotting_status(progress_key)
751 return status
752
753 @cherrypy.expose
754 @cherrypy.tools.json_out()
755 def revisions(self, revision_label=None):
756 """
757 Return a combined json object with all revisions and
758 mark the newest one with the field most_recent=true
759 """
760
761 # get list of available revision
762 rev_list = get_json_object_list(
764 validationpath.file_name_results_json,
765 )
766
767 # always add the reference revision
768 combined_list = []
769 reference_revision = json.loads(
770 json_objects.dumps(json_objects.Revision(label="reference"))
771 )
772
773 # load and combine
774 for r in rev_list:
775 full_path = os.path.join(
777 r,
778 validationpath.file_name_results_json,
779 )
780
781 # update label, if dir has been moved
782 lbl_folder = get_revision_label_from_json_filename(full_path)
783 j = deliver_json(full_path)
784 j["label"] = lbl_folder
785 combined_list.append(j)
786
787 # Sorting
788
789 # Order by categories (nightly, release, etc.) first, then by date
790 # A pure chronological order doesn't make sense, because we do not
791 # have a linear history ((pre)releases branch off) and for the builds
792 # the date corresponds to the build date, not to the date of the
793 # actual commit.
794 def sort_key(label: str):
795 if "-" not in label:
796 logging.warning(
797 f"Misformatted label encountered: '{label}' "
798 f"(doesn't seem to include date?)"
799 )
800 return label
801 category, datetag = label.split("-", maxsplit=1)
802 print(category, datetag)
803 # Will later reverse order to bring items in the same category
804 # in reverse chronological order, so the following list will have
805 # the items in reverse order as well:
806 order = ["release", "prerelease", "nightly"]
807 try:
808 index = order.index(category)
809 except ValueError:
810 index = 9
811 logging.warning(
812 f"Misformatted label encountered: '{label}' (doesn't seem "
813 f"to belong to any known category?)"
814 )
815 return f"{index}-{datetag}"
816
817 combined_list.sort(key=lambda rev: sort_key(rev["label"]), reverse=True)
818
819 # reference always on top
820 combined_list = [reference_revision] + combined_list
821
822 # Set the most recent one ...
823 newest_date = None
824 newest_rev = None
825 for r in combined_list:
826 rdate_str = r["creation_date"]
827 if isinstance(rdate_str, str):
828 if len(rdate_str) > 0:
829 try:
830 rdate = time.strptime(rdate_str, "%Y-%m-%d %H:%M")
831 except ValueError:
832 # some old validation results might still contain
833 # seconds and therefore cannot properly be converted
834 rdate = None
835
836 if rdate is None:
837 continue
838
839 if newest_date is None:
840 newest_date = rdate
841 newest_rev = r
842 if rdate > newest_date:
843 newest_date = rdate
844 newest_rev = r
845
846 for c in combined_list:
847 if c["most_recent"] is not None:
848 c["most_recent"] = False
849
850 # if there are no revisions at all, this might also be just None
851 if newest_rev:
852 newest_rev["most_recent"] = True
853
854 # topmost item must be dictionary for the ractive.os template to match
855 return {"revisions": combined_list}
856
857 @cherrypy.expose
858 @cherrypy.tools.json_out()
859 def comparisons(self, comparison_label=None):
860 """
861 return the json file of the comparison results of one specific
862 comparison
863 """
864
865 warn_wrong_directory()
866
867 # todo: Make this independent of our working directory!
868 path = os.path.join(
869 os.path.relpath(
871 self.working_folder, comparison_label.split(",")
872 ),
874 ),
875 "comparison.json",
876 )
877
878 # check if this comparison actually exists
879 if not os.path.isfile(path):
880 raise cherrypy.HTTPError(
881 404, f"Json Comparison file {path} does not exist"
882 )
883
884 return deliver_json(path)
885
886 @cherrypy.expose
887 @cherrypy.tools.json_out()
888 def system_info(self):
889 """
890 Returns:
891 JSON file containing git versions and time of last restart
892 """
893
894 warn_wrong_directory()
895
896 # note: for some reason %Z doesn't work like this, so we use
897 # time.tzname for the time zone.
898 return {
899 "last_restart": self.last_restart.strftime("%-d %b %H:%M ")
900 + time.tzname[1],
901 "version_restart": self.version,
903 os.environ["BELLE2_LOCAL_DIR"]
904 ),
905 }
906
907 @cherrypy.expose
908 def retrieve_file_metadata(self, filename):
909 """
910 Returns:
911 Metadata(str) of the file
912 """
913 cherrypy.response.headers['Content-Type'] = 'text/plain'
914 metadata = validationfunctions.get_file_metadata(filename)
915 return metadata
916
917 @cherrypy.expose
918 @cherrypy.tools.json_in()
919 @cherrypy.tools.json_out()
920 def create_issue(self, title, description):
921 """
922 Call the functions to create the issue and redirect
923 to the created Gitlab issue page.
924 """
925
926 # check if this is a plot/script based on file format
927 issue_type = 'plot'
928 if 'log' == self.file_path.split("/")[-1].split(".")[-1]:
929 issue_type = 'script'
930 default_section = self.gitlab_config['global']['default']
931 project_id = self.gitlab_config[default_section]['project_id']
932 # Create issue in the Gitlab project and save it
933 project = get_project_object(self.gitlab_object, project_id)
934 uploaded_file = upload_file_gitlab(self.file_path, project)
935 assignees = {
936 'gitlab_ids': [],
937 'usernames': [],
938 }
939 file_name = self.file_path.split("/")[-1].split(".log")[0]
940 file_package = self.file_path.split("/")[-2]
941 if self.gitlab_map:
942 assignees = parse_contact(
943 self.contact, self.gitlab_map, file_package, self.gitlab_object
944 )
945 description += "\n\n---\n\n:robot: Automated code, please do not delete\n\n" + \
946 f"Relevant {issue_type}: {file_name}\n\n" + \
947 f"Revision label: {self.revision_label}\n\n---"
948 issue_id = create_gitlab_issue(
949 title, description, uploaded_file, assignees, file_package, project
950 )
951 project.save()
952
953 # Update JSON with created issue id - script and plot info reside
954 # in different locations and also have different structures.
955 # todo - maybe this can be combined?
956 if issue_type == 'script':
957 revision_json_path = os.path.join(
959 self.revision_label,
960 validationpath.file_name_results_json,
961 )
962 update_scriptfile_issues_json(
963 revision_json_path, file_name, file_package, issue_id)
964 else:
965 comparison_json_path = os.path.join(
967 self.file_path.split("/")[-3],
968 "comparison.json",
969 )
970 update_plot_issues_json(
971 comparison_json_path, file_name, file_package, issue_id)
972
973 issue_url = self.gitlab_config[default_section]['project_url'] \
974 + "/-/issues/" \
975 + str(issue_id)
976 raise cherrypy.HTTPRedirect(
977 issue_url
978 )
979
980 @cherrypy.expose
981 def issue(self, file_path, rev_label, contact):
982 """
983 Return a template issue creation interface
984 for the user to add title and description.
985 """
986 self.file_path = os.path.join(
988 )
989
990 self.revision_label = rev_label
991 self.contact = contact
992
993 if not self.gitlab_object:
994 return "ERROR: Gitlab integration not set up, verify config file."
995
996 raise cherrypy.HTTPRedirect("/static/validation_issue.html")
997
998 @cherrypy.expose
999 def issue_redirect(self, iid):
1000 """
1001 Redirect to the Gitlab issue page.
1002 """
1003 default_section = self.gitlab_config['global']['default']
1004 if not self.gitlab_config[default_section]['project_url']:
1005 return "ERROR: Gitlab integration not set up, verify config file."
1006
1007 issue_url = self.gitlab_config[default_section]['project_url'] \
1008 + "/-/issues/" \
1009 + str(iid)
1010 raise cherrypy.HTTPRedirect(
1011 issue_url
1012 )
1013
1014 @cherrypy.expose
1015 def update_issue(self, id, file_path, rev_label):
1016 """
1017 Update existing issue in Gitlab with current result plot
1018 and redirect to the updated Gitlab issue page.
1019 """
1020
1021 if not self.gitlab_object:
1022 return "ERROR: Gitlab integration not set up, verify config file."
1023
1024 plot_path = os.path.join(
1026 )
1027
1028 default_section = self.gitlab_config['global']['default']
1029 project_id = self.gitlab_config[default_section]['project_id']
1030 project = get_project_object(self.gitlab_object, project_id)
1031 uploaded_file = upload_file_gitlab(plot_path, project)
1032 update_gitlab_issue(
1033 id, uploaded_file, project, plot_path, rev_label
1034 )
1035 project.save()
1036
1037 issue_url = self.gitlab_config[default_section]['project_url'] \
1038 + "/-/issues/" \
1039 + str(id)
1040
1041 raise cherrypy.HTTPRedirect(
1042 issue_url
1043 )
1044
1045
1046def setup_gzip_compression(path, cherry_config):
1047 """
1048 enable GZip compression for all text-based content the
1049 web-server will deliver
1050 """
1051
1052 cherry_config[path].update(
1053 {
1054 "tools.gzip.on": True,
1055 "tools.gzip.mime_types": [
1056 "text/html",
1057 "text/plain",
1058 "text/css",
1059 "application/javascript",
1060 "application/json",
1061 ],
1062 }
1063 )
1064
1065
1066def get_argument_parser():
1067 """
1068 Prepare a parser for all the known command line arguments
1069 """
1070
1071 # Set up the command line parser
1072 parser = argparse.ArgumentParser()
1073
1074 # Define the accepted command line flags and read them in
1075 parser.add_argument(
1076 "-ip",
1077 "--ip",
1078 help="The IP address on which the"
1079 "server starts. Default is '127.0.0.1'.",
1080 type=str,
1081 default="127.0.0.1",
1082 )
1083 parser.add_argument(
1084 "-p",
1085 "--port",
1086 help="The port number on which"
1087 " the server starts. Default is '8000'.",
1088 type=str,
1089 default=8000,
1090 )
1091 parser.add_argument(
1092 "-v",
1093 "--view",
1094 help="Open validation website" " in the system's default browser.",
1095 action="store_true",
1096 )
1097 parser.add_argument(
1098 "--production",
1099 help="Run in production environment: "
1100 "no log/error output via website and no auto-reload",
1101 action="store_true",
1102 )
1103 parser.add_argument(
1104 "-u",
1105 "--usermap",
1106 help="Path of file containing <email gitlab_username> map.",
1107 type=str,
1108 default=None,
1109 )
1110
1111 return parser
1112
1113
1114def parse_cmd_line_arguments():
1115 """!
1116 Sets up a parser for command line arguments,
1117 parses them and returns the arguments.
1118 @return: An object containing the parsed command line arguments.
1119 Arguments are accessed like they are attributes of the object,
1120 i.e. [name_of_object].[desired_argument]
1121 """
1122 parser = get_argument_parser()
1123 # Return the parsed arguments!
1124 return parser.parse_args()
1125
1126
1127def run_server(
1128 ip="127.0.0.1",
1129 port=8000,
1130 parse_command_line=False,
1131 open_site=False,
1132 dry_run=False,
1133):
1134
1135 # Setup options for logging
1136 logging.basicConfig(
1137 level=logging.DEBUG,
1138 format="%(asctime)s %(levelname)-8s %(message)s",
1139 datefmt="%H:%M:%S",
1140 )
1141
1142 basepath = validationpath.get_basepath()
1143 cwd_folder = os.getcwd()
1144
1145 # Only execute the program if a basf2 release is set up!
1146 if (
1147 os.environ.get("BELLE2_RELEASE_DIR", None) is None
1148 and os.environ.get("BELLE2_LOCAL_DIR", None) is None
1149 ):
1150 sys.exit("Error: No basf2 release set up!")
1151
1152 cherry_config = dict()
1153 # just empty, will be filled below
1154 cherry_config["/"] = {}
1155 # will ensure also the json requests are gzipped
1156 setup_gzip_compression("/", cherry_config)
1157
1158 # check if static files are provided via central release
1159 static_folder_list = ["validation", "html_static"]
1160 static_folder = None
1161
1162 if basepath["central"] is not None:
1163 static_folder_central = os.path.join(
1164 basepath["central"], *static_folder_list
1165 )
1166 if os.path.isdir(static_folder_central):
1167 static_folder = static_folder_central
1168
1169 # check if there is also a collection of static files in the local release
1170 # this overwrites the usage of the central release
1171 if basepath["local"] is not None:
1172 static_folder_local = os.path.join(
1173 basepath["local"], *static_folder_list
1174 )
1175 if os.path.isdir(static_folder_local):
1176 static_folder = static_folder_local
1177
1178 if static_folder is None:
1179 sys.exit(
1180 "Either BELLE2_RELEASE_DIR or BELLE2_LOCAL_DIR has to set "
1181 "to provide static HTML content. Did you run b2setup ?"
1182 )
1183
1184 # join the paths of the various result folders
1185 results_folder = validationpath.get_results_folder(cwd_folder)
1186 comparison_folder = validationpath.get_html_plots_folder(cwd_folder)
1187
1188 logging.info(f"Serving static content from {static_folder}")
1189 logging.info(f"Serving result content and plots from {cwd_folder}")
1190
1191 # check if the results folder exists and has at least one folder
1192 if not os.path.isdir(results_folder):
1193 sys.exit(
1194 f"Result folder {results_folder} does not exist, run validate_basf2 first " +
1195 "to create validation output"
1196 )
1197
1198 results_count = sum(
1199 [
1200 os.path.isdir(os.path.join(results_folder, f))
1201 for f in os.listdir(results_folder)
1202 ]
1203 )
1204 if results_count == 0:
1205 sys.exit(
1206 f"Result folder {results_folder} contains no folders, run "
1207 f"validate_basf2 first to create validation output"
1208 )
1209
1210 # Go to the html directory
1211 if not os.path.exists("html"):
1212 os.mkdir("html")
1213 os.chdir("html")
1214
1215 if not os.path.exists("plots"):
1216 os.mkdir("plots")
1217
1218 if os.path.exists("plots/rainbow.json"):
1219 logging.info("Removing old plots and unpopular combinations")
1221 comparison_folder,
1223 )
1224
1225 # export js, css and html templates
1226 cherry_config["/static"] = {
1227 "tools.staticdir.on": True,
1228 # only serve js, css, html and png files
1229 "tools.staticdir.match": r"^.*\.(js|css|html|png|js.map)$",
1230 "tools.staticdir.dir": static_folder,
1231 }
1232 setup_gzip_compression("/static", cherry_config)
1233
1234 # export generated plots
1235 cherry_config["/plots"] = {
1236 "tools.staticdir.on": True,
1237 # only serve json and png files
1238 "tools.staticdir.match": r"^.*\.(png|json|pdf)$",
1239 "tools.staticdir.dir": comparison_folder,
1240 }
1241 setup_gzip_compression("/plots", cherry_config)
1242
1243 # export generated results and raw root files
1244 cherry_config["/results"] = {
1245 "tools.staticdir.on": True,
1246 "tools.staticdir.dir": results_folder,
1247 # only serve root, log and txt files
1248 "tools.staticdir.match": r"^.*\.(log|root|txt)$",
1249 # server the log files as plain text files, and make sure to use
1250 # utf-8 encoding. Firefox might decide different, if the files
1251 # are located on a .jp domain and use Shift_JIS
1252 "tools.staticdir.content_types": {
1253 "log": "text/plain; charset=utf-8",
1254 "root": "application/octet-stream",
1255 },
1256 }
1257
1258 setup_gzip_compression("/results", cherry_config)
1259
1260 # Define the server address and port
1261 # only if we got some specific
1262 production_env = False
1263 if parse_command_line:
1264 # Parse command line arguments
1265 cmd_arguments = parse_cmd_line_arguments()
1266
1267 ip = cmd_arguments.ip
1268 port = int(cmd_arguments.port)
1269 open_site = cmd_arguments.view
1270 production_env = cmd_arguments.production
1271 usermap_file = cmd_arguments.usermap
1272
1273 cherrypy.config.update(
1274 {
1275 "server.socket_host": ip,
1276 "server.socket_port": port,
1277 }
1278 )
1279 if production_env:
1280 cherrypy.config.update({"environment": "production"})
1281
1282 logging.info(f"Server: Starting HTTP server on {ip}:{port}")
1283
1284 if open_site:
1285 webbrowser.open("http://" + ip + ":" + str(port))
1286
1287 config_path = os.path.join(static_folder, '../config/gl.cfg')
1288
1289 if not dry_run:
1290 # gitlab toggle
1291 gitlab_object = None
1292 gitlab_config = None
1293 gitlab_map = None
1294 if not os.path.exists(config_path):
1295 logging.warning(
1296 "ERROR: Expected to find config folder with Gitlab config,"
1297 f" but {config_path} doesn't exist. "
1298 "Gitlab features will not work."
1299 )
1300 else:
1301 gitlab_config = get_gitlab_config(config_path)
1302 gitlab_object = create_gitlab_object(config_path)
1303 if gitlab_object:
1304 update_linked_issues(gitlab_object, cwd_folder)
1305 gitlab_map = usermap_file
1306 logging.info(
1307 f"{gitlab_map} will be used to assign issues."
1308 )
1309
1310 cherrypy.quickstart(
1312 working_folder=cwd_folder,
1313 gitlab_object=gitlab_object,
1314 gitlab_config=gitlab_config,
1315 gitlab_map=gitlab_map,
1316 ),
1317 "/",
1318 cherry_config,
1319 )
1320
1321
1322if __name__ == "__main__":
1323 run_server()
create_issue(self, title, description)
update_issue(self, id, file_path, rev_label)
contact
placeholder variable for contact
last_restart
Date when this object was instantiated.
revisions(self, revision_label=None)
comparisons(self, comparison_label=None)
file_path
placeholder variable for path
issue(self, file_path, rev_label, contact)
working_folder
html folder that contains plots etc.
__init__(self, working_folder, gitlab_object, gitlab_config, gitlab_map)
revision_label
placeholder variable for revision label
clear_plots(str work_folder, List[str] keep_revisions)
Optional[str] get_compact_git_hash(str repo_folder)
str get_file_metadata(str filename)
List[str] get_popular_revision_combinations(str work_folder)
get_html_folder(output_base_dir)
get_html_plots_folder(output_base_dir)
get_html_plots_tag_comparison_folder(output_base_dir, tags)
get_results_folder(output_base_dir)