From 34c6700b34d817e53f0613e7c9316ccaa6c64486 Mon Sep 17 00:00:00 2001 From: AdrianoDee Date: Wed, 15 May 2024 14:40:56 +0200 Subject: [PATCH 1/7] Fix RelMon piecharts with Chart.js --- Utilities/RelMon/python/directories2html.py | 33 +++++++++---------- Utilities/RelMon/python/dirstructure.py | 35 +++++++++++++++++++++ 2 files changed, 52 insertions(+), 16 deletions(-) diff --git a/Utilities/RelMon/python/directories2html.py b/Utilities/RelMon/python/directories2html.py index 73c769bb96041..5304258524c73 100755 --- a/Utilities/RelMon/python/directories2html.py +++ b/Utilities/RelMon/python/directories2html.py @@ -100,6 +100,7 @@ def get_page_header(directory=None, standalone=False, additional_header=""): html=''+\ ''+\ 'RelMon Summary'+\ + '' + \ ''%style_location+\ ''%style_location+\ ''%style_location+\ @@ -212,7 +213,7 @@ def get_subdirs_section(directory, hashing_flag): html+='' html+='
' - html+=''%(link,subdir.get_summary_chart_ajax(150,100)) + html+= subdir.get_piechart_js(150,link) html+='
' html+='
' @@ -263,7 +264,7 @@ def get_summary_section(directory,matrix_page=True): html+='' html+='
'+\ - ''%directory.get_summary_chart_ajax(200,200)+\ + '%s'%(directory.get_piechart_js(200)) +\ '
'+\ '
' if matrix_page: @@ -847,16 +848,17 @@ def make_summary_table(indir,aggregation_rules,aggregation_rules_twiki, hashing_ page_html+='
' page_html+='Summary
' - page_html+='
%s
'%(global_dir.get_summary_chart_ajax(55,55),get_pie_tooltip(global_dir)) + page_html+='
%s
'%(global_dir.get_piechart_js(55)) for sample in sorted_samples: col=dir_dict[sample] # check if the directory was a top one or not summary_page_name="RelMonSummary.html" if col.name!="": summary_page_name=hash_name(col.name, hashing_flag)+".html" - img_link=col.get_summary_chart_ajax(55,55) + title = get_pie_tooltip(col) + chart = col.get_piechart_js(55,sample+"/"+summary_page_name,title) page_html+='
' - page_html+='
' %(sample,summary_page_name,img_link,get_pie_tooltip(col)) + page_html+='%s
' %(chart) page_html+="" # Now the content @@ -878,9 +880,10 @@ def make_summary_table(indir,aggregation_rules,aggregation_rules_twiki, hashing_ # one first row for the summary! row_summary.calcStats() - img_link=row_summary.get_summary_chart_ajax(55,55) + title = get_pie_tooltip(col) + chart = row_summary.get_piechart_js(55,None,title) page_html+='
' - page_html+='
' %(img_link,get_pie_tooltip(row_summary)) + page_html+= chart + '' for sample in sorted_samples: sample_counter+=1 @@ -903,19 +906,17 @@ def make_summary_table(indir,aggregation_rules,aggregation_rules_twiki, hashing_ summary_page=join(sample,"%s.html"%(hash_name(directory.name+subdir_name,hashing_flag))) dir_is_there=subdir_name in subdirs_dict - img_link="https://chart.googleapis.com/chart?cht=p3&chco=C0C0C0&chs=50x50&chd=t:1" + img_link="https://upload.wikimedia.org/wikipedia/commons/a/a8/Circle_Davys-Grey_Solid.svg" img_tooltip="N/A" + page_html+='
' + if dir_is_there: #row_summary.subdirs.append(subdirs_dict[subdir_name]) - img_link=subdirs_dict[subdir_name].get_summary_chart_ajax(50,50) img_tooltip=get_pie_tooltip(subdirs_dict[subdir_name]) - - page_html+='
' - if dir_is_there: - page_html+=''%(summary_page) - page_html+='' %(img_link,img_tooltip) - if dir_is_there: - page_html+='' + chart=subdirs_dict[subdir_name].get_piechart_js(50,summary_page,img_tooltip) + page_html+='%s'%chart + else: + page_html+='' %(img_link,img_tooltip) page_html+='
' page_html+=" \n" diff --git a/Utilities/RelMon/python/dirstructure.py b/Utilities/RelMon/python/dirstructure.py index 7fb72841799c9..d90ca82a93440 100755 --- a/Utilities/RelMon/python/dirstructure.py +++ b/Utilities/RelMon/python/dirstructure.py @@ -15,6 +15,7 @@ from copy import deepcopy from os import chdir,getcwd,listdir,makedirs,rmdir from os.path import exists,join +import random import sys argv=sys.argv @@ -180,6 +181,40 @@ def get_summary_chart_ajax(self,w=400,h=300): return url + def get_piechart_js(self,w=400,link=None,title=None): + """ + Build the HTML snippet to render a piechart with chart.js + """ + + name = random.getrandbits(64) # just a random has for the canvas + html = "" + html += ''%(name,w) + # piechart + html += '' + + return html + def print_report(self,indent="",verbose=False): if len(indent)==0: self.calcStats(make_pie=False) From 50d9cff3634b785e67abccbbb0cf021106f71527 Mon Sep 17 00:00:00 2001 From: AdrianoDee Date: Thu, 16 May 2024 13:44:35 +0200 Subject: [PATCH 2/7] Add Chart.js gauge plots --- Utilities/RelMon/python/definitions.py | 5 ++ Utilities/RelMon/python/directories2html.py | 60 +++++++++++++-------- Utilities/RelMon/python/dirstructure.py | 15 +----- 3 files changed, 45 insertions(+), 35 deletions(-) diff --git a/Utilities/RelMon/python/definitions.py b/Utilities/RelMon/python/definitions.py index 4e9c8f40d17f9..53240d312d124 100755 --- a/Utilities/RelMon/python/definitions.py +++ b/Utilities/RelMon/python/definitions.py @@ -197,3 +197,8 @@ ("!(mu20+|wzMu20+|jet20+)","Btag@1")) data_pattern_blist_pairs=() +## colors for gauge + +from matplotlib.colors import LinearSegmentedColormap +gauge_cmap=LinearSegmentedColormap.from_list('rg',["r", "orange","y","lime"], N=256) + diff --git a/Utilities/RelMon/python/directories2html.py b/Utilities/RelMon/python/directories2html.py index 5304258524c73..fc0c4227fd18d 100755 --- a/Utilities/RelMon/python/directories2html.py +++ b/Utilities/RelMon/python/directories2html.py @@ -24,6 +24,7 @@ import os import hashlib +import random if "RELMON_SA" in os.environ: from .dirstructure import Comparison,Directory @@ -492,27 +493,44 @@ def directory2html(directory, hashing, standalone, depth=0): #chdir(old_cwd) -#------------------------------------------------------------------------------- +def to_rgba(t): + ''' Convert a RGBA tuple to string''' + return "rgba(%f,%f,%f,%f)"%(t) + +def build_gauge_js(rate,w=100,minrate=.80): + + color = to_rgba(gauge_cmap(rate-minrate)) + font_size = int(w/5) + text_offs = int(w/8) + gauge_max = 1. - rate + + name = random.getrandbits(64) # just a random has for the canvas + html = "" + html += '
'%(w) + html += '
'%(text_offs) + html += '%.2f%%'%(color,font_size,rate*100.) + html += ''%(name) + + # "gauge" chart + html += '
' + + print("here") + + img_link="https://upload.wikimedia.org/wikipedia/commons/a/a8/Circle_Davys-Grey_Solid.svg" + html='' %(img_link,"test") + + return html #------------------------------------------------------------------------------- def get_aggr_pairs_info(dir_dict,the_aggr_pairs=[]): @@ -615,7 +633,7 @@ def make_categories_summary(dir_dict,aggregation_rules): html+='
' html+='
' - html+=build_gauge(average_success_rate) + html+=build_gauge_js(average_success_rate) html+='
' html+='
' @@ -649,7 +667,7 @@ def make_twiki_table(dir_dict,aggregation_rules): for cat_name,present_subdirs,total_weight,average_success_rate in aggr_pairs_info: #print cat_name,present_subdirs,total_weight,average_success_rate - html+=build_gauge(average_success_rate,small=True,escaped=True) + html+=build_gauge_js(average_success_rate,w=40) html+=" | " html+=' Top...' diff --git a/Utilities/RelMon/python/dirstructure.py b/Utilities/RelMon/python/dirstructure.py index d90ca82a93440..e4b9557d6a864 100755 --- a/Utilities/RelMon/python/dirstructure.py +++ b/Utilities/RelMon/python/dirstructure.py @@ -168,19 +168,6 @@ def get_subdirs_names(self): subdirnames.append(subdir.name) return subdirnames - def get_summary_chart_ajax(self,w=400,h=300): - """Emit the ajax to build a pie chart using google apis... - """ - url = "https://chart.googleapis.com/chart?" - url+= "cht=p3" # Select the 3d chart - #url+= "&chl=Success|Null|Fail" # give labels - url+= "&chco=00FF00|FFFF00|FF0000|7A7A7A" # give colours to labels - url+= "&chs=%sx%s" %(w,h) - #url+= "&chtt=%s" %self.name - url+= "&chd=t:%.2f,%.2f,%.2f,%.2f"%(self.get_success_rate(),self.get_null_rate(),self.get_fail_rate(),self.get_skiped_rate()) - - return url - def get_piechart_js(self,w=400,link=None,title=None): """ Build the HTML snippet to render a piechart with chart.js @@ -188,7 +175,7 @@ def get_piechart_js(self,w=400,link=None,title=None): name = random.getrandbits(64) # just a random has for the canvas html = "" - html += ''%(name,w) + html += ''%(name,w) # piechart html += ' ' - - print("here") - - img_link="https://upload.wikimedia.org/wikipedia/commons/a/a8/Circle_Davys-Grey_Solid.svg" - html='' %(img_link,"test") + html += 'options: { responsive: true, rotation: -3.1415926536, circumference: 3.1415926536 ,' ## in radiants + html += ' legend: { display: false }, tooltips: {enabled: false}, hover: {mode: null}},' + html += '}); ' + if add_rate: + html += '
'%(int(2*w/7),font_size) + html += '%.2f%%
'%(color,font_size,rate*100.) return html #------------------------------------------------------------------------------- @@ -633,7 +640,7 @@ def make_categories_summary(dir_dict,aggregation_rules): html+='' html+='
' - html+=build_gauge_js(average_success_rate) + html+=build_gauge_js(average_success_rate,add_rate=True) html+='
' html+='
' @@ -710,7 +717,7 @@ def make_barchart_summary(dir_dict,name="the_chart",title="DQM directory",the_ag }); } - """%(name,40*counter,title) + """%(name,35*counter,title) return script @@ -821,7 +828,7 @@ def make_summary_table(indir,aggregation_rules,aggregation_rules_twiki, hashing_ page_html+=make_categories_summary(dir_dict,aggregation_rules) # Make the Directories chart - page_html+='

Detailed Barchart

' + # page_html+='

Detailed Barchart

' page_html+='
Top...
' # Barbarian vertical space. Suggestions are welcome @@ -830,24 +837,31 @@ def make_summary_table(indir,aggregation_rules,aggregation_rules_twiki, hashing_ # Prepare the table - page_html+='

Summary Table

' + page_html+='

Summary Table

(scrollable)

' - for i in range(5): - page_html+='

\n' - + # for i in range(5): + # page_html+='

\n' + + div_width= min(len(dir_dict.keys()) * 70 + 500,1500) #80 px per column + 200 for the first column + page_html+='
'%(div_width) page_html+=""" - +
+ - + """ # First row with samples page_html+=""" - """ + """ sorted_samples=sorted(dir_dict.keys()) for sample in sorted_samples: - sample_nick=sample + if "_" in sample: + sample_nick="_".join(sample.split("X_")[0].split("_")[:-1]) + #slightly cleaner: _X is for the GT string, the _ split take away the 123X + else: + sample_nick = sample ## For runs: put only the number after the _ #if "_" in sample: #run_number=sample.split("_")[-1] @@ -856,17 +870,17 @@ def make_summary_table(indir,aggregation_rules,aggregation_rules_twiki, hashing_ page_html+=""" - """%sample_nick - page_html+=" \n" +

"""%sample_nick + page_html+=" \n \n \n" # FIRST ROW # Now the summaries at the beginning of the table page_html+="" - page_html+='' - page_html+=''%(global_dir.get_piechart_js(55)) + page_html+='Summary' + page_html+=''%(global_dir.get_piechart_js(50)) for sample in sorted_samples: col=dir_dict[sample] # check if the directory was a top one or not @@ -874,16 +888,16 @@ def make_summary_table(indir,aggregation_rules,aggregation_rules_twiki, hashing_ if col.name!="": summary_page_name=hash_name(col.name, hashing_flag)+".html" title = get_pie_tooltip(col) - chart = col.get_piechart_js(55,sample+"/"+summary_page_name,title) - page_html+='' %(chart) + chart = col.get_piechart_js(50,sample+"/"+summary_page_name) + page_html+='' %(chart) page_html+="" # Now the content for subdir_name in all_subdirs: page_html+=' \n' - page_html+=' \n' %subdir_name + page_html+=' \n' %subdir_name row_summary=Directory("row_summary","") sample_counter=0 @@ -899,8 +913,8 @@ def make_summary_table(indir,aggregation_rules,aggregation_rules_twiki, hashing_ # one first row for the summary! row_summary.calcStats() title = get_pie_tooltip(col) - chart = row_summary.get_piechart_js(55,None,title) - page_html+='' for sample in sorted_samples: @@ -925,27 +939,26 @@ def make_summary_table(indir,aggregation_rules,aggregation_rules_twiki, hashing_ dir_is_there=subdir_name in subdirs_dict img_link="https://upload.wikimedia.org/wikipedia/commons/a/a8/Circle_Davys-Grey_Solid.svg" - img_tooltip="N/A" - page_html+='' page_html+=" \n" - page_html+='

Summary

Summary

%s

%s

' + page_html+='
' - page_html+='Summary
%s
%s
' - page_html+='%s
' + page_html+='%s
%s%s
' + chart = row_summary.get_piechart_js(50) + page_html+='
'#
' page_html+= chart + '
' + page_html+='
' if dir_is_there: #row_summary.subdirs.append(subdirs_dict[subdir_name]) - img_tooltip=get_pie_tooltip(subdirs_dict[subdir_name]) - chart=subdirs_dict[subdir_name].get_piechart_js(50,summary_page,img_tooltip) + chart=subdirs_dict[subdir_name].get_piechart_js(50,summary_page) page_html+='%s'%chart else: - page_html+='' %(img_link,img_tooltip) + page_html+='' %(img_link) page_html+='
Top...
' + page_html+='
Top...
' page_html+=get_rank_section(global_dir) - page_html+=make_twiki_table(dir_dict,aggregation_rules_twiki) + #page_html+=make_twiki_table(dir_dict,aggregation_rules_twiki) + # ^ commenting out for the moment, not really useful nor used page_html+=get_page_footer() return page_html diff --git a/Utilities/RelMon/python/dirstructure.py b/Utilities/RelMon/python/dirstructure.py index e4b9557d6a864..a2b407d5aa159 100755 --- a/Utilities/RelMon/python/dirstructure.py +++ b/Utilities/RelMon/python/dirstructure.py @@ -168,14 +168,21 @@ def get_subdirs_names(self): subdirnames.append(subdir.name) return subdirnames - def get_piechart_js(self,w=400,link=None,title=None): + def get_piechart_js(self,w=400,link=None): + """ Build the HTML snippet to render a piechart with chart.js """ + if self.get_success_rate()>=99.9: # if the success rate is very high let's make the page lighter + img_link = "https://raw.githubusercontent.com/cms-PdmV/RelMonService2/5ee98db210c0898fd34b4deac3653fa2bdff269b/report_website/lime_circle.png" + html ='' %(img_link,w,w) + if link is not None: + html = ' %s ' %(link,html) + return html name = random.getrandbits(64) # just a random has for the canvas html = "" - html += ''%(name,w) + html += ''%(name,w,w) # piechart html += '' From 90970d1a66cb71badfa99af2ddc8000fae375d10 Mon Sep 17 00:00:00 2001 From: AdrianoDee Date: Mon, 27 May 2024 11:27:19 +0200 Subject: [PATCH 4/7] Fix naming for Data RelVals --- Utilities/RelMon/python/directories2html.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/Utilities/RelMon/python/directories2html.py b/Utilities/RelMon/python/directories2html.py index b24bec29c4d14..19d7246b0ea4d 100755 --- a/Utilities/RelMon/python/directories2html.py +++ b/Utilities/RelMon/python/directories2html.py @@ -828,7 +828,6 @@ def make_summary_table(indir,aggregation_rules,aggregation_rules_twiki, hashing_ page_html+=make_categories_summary(dir_dict,aggregation_rules) # Make the Directories chart - # page_html+='

Detailed Barchart

' page_html+='
Top...
' # Barbarian vertical space. Suggestions are welcome @@ -839,9 +838,6 @@ def make_summary_table(indir,aggregation_rules,aggregation_rules_twiki, hashing_ # Prepare the table page_html+='

Summary Table

(scrollable)

' - # for i in range(5): - # page_html+='

\n' - div_width= min(len(dir_dict.keys()) * 70 + 500,1500) #80 px per column + 200 for the first column page_html+='
'%(div_width) page_html+=""" @@ -857,17 +853,14 @@ def make_summary_table(indir,aggregation_rules,aggregation_rules_twiki, hashing_ sorted_samples=sorted(dir_dict.keys()) for sample in sorted_samples: - if "_" in sample: + if "_" in sample and "Data" not in sample: sample_nick="_".join(sample.split("X_")[0].split("_")[:-1]) - #slightly cleaner: _X is for the GT string, the _ split take away the 123X + # Cleaning for MC: just the fragment + elif "Data" in sample and "RelVal" in sample: + sample_nick = "".join([sample.split("_")[0],sample.split("RelVal")[-1]]) + # Cleaning for Data: PD + Era + Run else: sample_nick = sample - ## For runs: put only the number after the _ - #if "_" in sample: - #run_number=sample.split("_")[-1] - #if (not run_number.isalpha()) and len(run_number)>=6: - #sample_nick=run_number - page_html+="""

%s

"""%sample_nick From 704f07266f3286035a5a96d7695dcc0f96c553fb Mon Sep 17 00:00:00 2001 From: AdrianoDee Date: Thu, 6 Jun 2024 13:11:36 +0200 Subject: [PATCH 5/7] Fix Grey Solid --- Utilities/RelMon/python/directories2html.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Utilities/RelMon/python/directories2html.py b/Utilities/RelMon/python/directories2html.py index 19d7246b0ea4d..ce7f2b66783f4 100755 --- a/Utilities/RelMon/python/directories2html.py +++ b/Utilities/RelMon/python/directories2html.py @@ -939,7 +939,7 @@ def make_summary_table(indir,aggregation_rules,aggregation_rules_twiki, hashing_ chart=subdirs_dict[subdir_name].get_piechart_js(50,summary_page) page_html+='%s'%chart else: - page_html+='' %(img_link) + page_html+='' %(img_link,"Unavailable") page_html+='
' page_html+=" \n" From 6ae61cf8aa22289938db5ee45211e712ef3c2a4b Mon Sep 17 00:00:00 2001 From: AdrianoDee Date: Fri, 7 Jun 2024 11:35:51 +0200 Subject: [PATCH 6/7] Move stylesheets link, fix grey circle. --- Utilities/RelMon/python/directories2html.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Utilities/RelMon/python/directories2html.py b/Utilities/RelMon/python/directories2html.py index ce7f2b66783f4..9ba5ffe388727 100755 --- a/Utilities/RelMon/python/directories2html.py +++ b/Utilities/RelMon/python/directories2html.py @@ -83,7 +83,7 @@ def fairy_url_single(run,sample,version,plot_path,tier,draw_opts="",h=250,w=200) def get_page_header(directory=None, standalone=False, additional_header=""): style_location="/cms-service-reldqm" if standalone: - style_location = "http://cms-service-reldqm.web.cern.ch/" + style_location +"/" + style_location = "https://raw.githubusercontent.com/cms-PdmV/RelMonService2/77c534ec93401ca5de222ac62a6422f02389dafc/report_website/" #RelMonService2 javascripts='' style='' tablestyle='' From 23e0cc452c92f359189c979cebed82c592fe255f Mon Sep 17 00:00:00 2001 From: Adriano Di Florio Date: Fri, 7 Jun 2024 16:26:26 +0200 Subject: [PATCH 7/7] Fixing path for stylesheets Co-authored-by: Dimitris Papagiannis --- Utilities/RelMon/python/directories2html.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Utilities/RelMon/python/directories2html.py b/Utilities/RelMon/python/directories2html.py index 9ba5ffe388727..c15509f4cf249 100755 --- a/Utilities/RelMon/python/directories2html.py +++ b/Utilities/RelMon/python/directories2html.py @@ -120,9 +120,9 @@ def get_page_header(directory=None, standalone=False, additional_header=""): ''+\ 'RelMon Summary'+\ '' + \ - ''%style_location+\ - ''%style_location+\ - ''%style_location+\ + ''%style_location+\ + ''%style_location+\ + ''%style_location+\ '