Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[14_0_X] Fix RelMon Piecharts using Chart.js #45256

Merged
merged 7 commits into from
Jun 24, 2024
Merged

[14_0_X] Fix RelMon Piecharts using Chart.js #45256

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
34c6700
Fix RelMon piecharts with Chart.js
May 15, 2024
50d9cff
Add Chart.js gauge plots
May 16, 2024
ecfad39
Improvements to readability of the table and making this lighter
May 21, 2024
90970d1
Fix naming for Data RelVals
May 27, 2024
704f072
Fix Grey Solid
Jun 6, 2024
6ae61cf
Move stylesheets link, fix grey circle.
Jun 7, 2024
23e0cc4
Fixing path for stylesheets
AdrianoDee Jun 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions Utilities/RelMon/python/definitions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -197,3 +197,11 @@
("!(mu20+|wzMu20+|jet20+)","Btag@1"))
data_pattern_blist_pairs=()

## colors for gauge
from matplotlib.colors import LinearSegmentedColormap
gauge_cmap=LinearSegmentedColormap.from_list('rg',["r", "orange","y","lime"], N=256)

## cms logo
cms_logo_url = "https://cms-docdb.cern.ch/cgi-bin/PublicDocDB/RetrieveFile?docid=3045&filename=CMSlogo_color_label_1024_May2014.png&version=3"


171 changes: 98 additions & 73 deletions Utilities/RelMon/python/directories2html.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@

import os
import hashlib
import random
from matplotlib.colors import to_hex

if "RELMON_SA" in os.environ:
from .dirstructure import Comparison,Directory
Expand Down Expand Up @@ -81,9 +83,12 @@ def fairy_url_single(run,sample,version,plot_path,tier,draw_opts="",h=250,w=200)
def get_page_header(directory=None, standalone=False, additional_header=""):
style_location="/cms-service-reldqm"
if standalone:
style_location = "http://cms-service-reldqm.web.cern.ch/" + style_location +"/"
style_location = "https://raw.githubusercontent.com/cms-PdmV/RelMonService2/77c534ec93401ca5de222ac62a6422f02389dafc/report_website/" #RelMonService2
javascripts=''
style=''
tablestyle=''
thead_h = 400
wrapper_h = 1500
if directory!=None and len(directory.comparisons)>0:
meta=directory.meta
style='img.fail {border:1px solid #ff0000;}\n'+\
Expand All @@ -94,18 +99,34 @@ def get_page_header(directory=None, standalone=False, additional_header=""):
'a.black_link:hover {color: #737373}\n'+\
'a.black_link:visited {color: #333333}\n'+\
'a.black_link:active {color: #333333}\n'
## fixed first row and first column table

wrapper_h = min(thead_h + (70 * len(directory.comparisons)),1800)

tablestyle = '\n.wrapper { overflow: auto; height: %dpx;} \n'%(wrapper_h)
tablestyle += 'table { position: relative; border-collapse: separate; border-spacing: 0;} \n'
tablestyle += 'table { position: relative; border-collapse: separate; border-spacing: 0;} \n'
tablestyle += 'table th, table td { width: 50px; padding: 5px; background-color: white;} \n'
tablestyle += 'table th { position: sticky; top: 0; z-index: 2; height: %dpx;} \n'%(thead_h)
tablestyle += 'table th:nth-child(1) { left:0; z-index:3;} \n'
tablestyle += '.sticky-col { position: sticky; background-color: #C9FFD1 ; width: 200px; left:0}\n'
tablestyle += '.center_head { position: absolute; top: 50%; left: 50%;} \n'
tablestyle += '.vertical_head {top: 60%; -webkit-transform: translateX(-50%) translateY(-50%) rotate(-90deg); -moz-transform: translateX(-50%) translateY(-50%) rotate(-90deg);} \n'
javascripts=""



html='<html>'+\
'<head>'+\
'<title>RelMon Summary</title>'+\
'<link rel="stylesheet" href="%s/style/blueprint/screen.css" type="text/css" media="screen, projection">'%style_location+\
'<link rel="stylesheet" href="%s/style/blueprint/print.css" type="text/css" media="print">'%style_location+\
'<link rel="stylesheet" href="%s/style/blueprint/plugins/fancy-type/screen.css" type="text/css" media="screen, projection">'%style_location+\
'<script src="https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.9.4/Chart.js"></script>' + \
'<link rel="stylesheet" href="%s/screen.css" type="text/css" media="screen, projection">'%style_location+\
'<link rel="stylesheet" href="%s/print.css" type="text/css" media="print">'%style_location+\
'<link rel="stylesheet" href="%s/fancy-type-screen.css" type="text/css" media="screen, projection">'%style_location+\
'<style type="text/css">'+\
'.rotation {display: block;-webkit-transform: rotate(-90deg);-moz-transform: rotate(-90deg); }'+\
'%s'%style+\
'%s'%tablestyle +\
'</style>'+\
'%s'%javascripts+\
'%s'%additional_header+\
Expand All @@ -123,10 +144,7 @@ def get_page_footer():
#-------------------------------------------------------------------------------

def get_title_section(directory, hashing_flag, standalone, depth=2):
if standalone:
cms_logo_url = "http://cms-service-reldqm.web.cern.ch/cms-service-reldqm/style/CMS.gif"
else:
cms_logo_url = "cms-service-reldqm/style/CMS.gif"

mother_name=basename(directory.mother_dir)
mother_file_name=""
if depth==1:
Expand Down Expand Up @@ -212,7 +230,7 @@ def get_subdirs_section(directory, hashing_flag):
html+='</div>'

html+='<div class="span-6 last">'
html+='<a href="%s"><img src="%s" class="top right"></a>'%(link,subdir.get_summary_chart_ajax(150,100))
html+= subdir.get_piechart_js(120,link)
html+='</div>'

html+='<hr>'
Expand Down Expand Up @@ -263,7 +281,7 @@ def get_summary_section(directory,matrix_page=True):
html+='</div>'

html+='<div class="span-7 colborder">'+\
'<img src="%s" class="top right">'%directory.get_summary_chart_ajax(200,200)+\
'%s'%(directory.get_piechart_js(150)) +\
'</div>'+\
'<div class="span-9 last">'
if matrix_page:
Expand Down Expand Up @@ -491,27 +509,35 @@ def directory2html(directory, hashing, standalone, depth=0):

#chdir(old_cwd)

#-------------------------------------------------------------------------------
def build_gauge_js(rate,w=100,minrate=.80,add_rate=False):

color = to_hex(gauge_cmap((rate-minrate)/(1.0-minrate)))
font_size = int(w/9)
gauge_max = 1. - rate

name = random.getrandbits(64) # just a random has for the canvas
html = ""
html += '<canvas id="%s" style="width:100%%;max-width:%d"></canvas>'%(name,w)

# "gauge" chart
html += '<script> new Chart("%s",'%(name)
html += '{ type: "doughnut",'

def build_gauge(total_success_rate,minrate=.80,small=False,escaped=False):
total_success_rate_scaled=(total_success_rate-minrate)
total_success_rate_scaled_repr=total_success_rate_scaled/(1-minrate)
if total_success_rate_scaled_repr<0:
total_success_rate_scaled_repr=0
size_s="200x100"
if small:
size_s="40x30"
#print "Total success rate %2.2f and scaled %2.2f "%(total_success_rate,total_success_rate_scaled)
gauge_link ="https://chart.googleapis.com/chart?chs=%s&cht=gom"%size_s
gauge_link+="&chd=t:%2.1f"%(total_success_rate_scaled_repr*100.)
if not small:
gauge_link+="&chxt=x,y&chxl=0:|%2.1f%%|1:|%i%%|%i%%|100%%"%(total_success_rate*100,minrate*100.,(1+minrate)*50)
gauge_link+="&chma=10,10,10,0"
img_tag= '<img src="%s">'%gauge_link
if escaped:
img_tag=html.escape(img_tag)
return img_tag
# data
html += 'data: {'
html += 'labels: ["Success", "Failure"],'
html += 'datasets: [{ backgroundColor: ["%s", "#C7C7C7"],'%(color)
html += 'data: [%f,%f]}] },'%(rate,gauge_max)

#options
html += 'options: { responsive: true, rotation: -3.1415926536, circumference: 3.1415926536 ,' ## in radiants
html += ' legend: { display: false }, tooltips: {enabled: false}, hover: {mode: null}},'
html += '}); </script>'
if add_rate:
html += '<div style="width: 100%%; position: relative; left: %d; margin-top: -%dpx; font-align: center">'%(int(2*w/7),font_size)
html += '<span style="color: %s; font-family: courier; font-size: %dpx;">%.2f%%</span></div>'%(color,font_size,rate*100.)

return html
#-------------------------------------------------------------------------------

def get_aggr_pairs_info(dir_dict,the_aggr_pairs=[]):
Expand Down Expand Up @@ -614,7 +640,7 @@ def make_categories_summary(dir_dict,aggregation_rules):
html+='</div>'

html+='<div class="span-6 last">'
html+=build_gauge(average_success_rate)
html+=build_gauge_js(average_success_rate,add_rate=True)
html+='</div>'

html+='<hr>'
Expand Down Expand Up @@ -648,7 +674,7 @@ def make_twiki_table(dir_dict,aggregation_rules):

for cat_name,present_subdirs,total_weight,average_success_rate in aggr_pairs_info:
#print cat_name,present_subdirs,total_weight,average_success_rate
html+=build_gauge(average_success_rate,small=True,escaped=True)
html+=build_gauge_js(average_success_rate,w=40)
html+=" | "

html+='</div> <a href="#top">Top...</a>'
Expand Down Expand Up @@ -691,7 +717,7 @@ def make_barchart_summary(dir_dict,name="the_chart",title="DQM directory",the_ag
});
}
</script>
"""%(name,40*counter,title)
"""%(name,35*counter,title)
return script


Expand Down Expand Up @@ -802,7 +828,6 @@ def make_summary_table(indir,aggregation_rules,aggregation_rules_twiki, hashing_
page_html+=make_categories_summary(dir_dict,aggregation_rules)

# Make the Directories chart
page_html+='<div class="span-24"><h2 class="alt"><a name="detailed_barchart">Detailed Barchart</a></h2></div>'
page_html+='<div id="dir_chart"></div> <a href="#top">Top...</a><hr>'

# Barbarian vertical space. Suggestions are welcome
Expand All @@ -811,59 +836,61 @@ def make_summary_table(indir,aggregation_rules,aggregation_rules_twiki, hashing_


# Prepare the table
page_html+='<div class="span-24"><h2 class="alt"><a name="summary_table">Summary Table</a></h2></div>'
page_html+='<div class="span-24"><h2 class="alt"><a name="summary_table">Summary Table</a></h2> <h4> <span class="alt"> (scrollable) </span> </h4> </div>'

for i in range(5):
page_html+='<div class="span-24"><p></p></div>\n'

div_width= min(len(dir_dict.keys()) * 70 + 500,1500) #80 px per column + 200 for the first column
page_html+='<div class="wrapper" style = "width: %dpx;">'%(div_width)
page_html+="""
<table border="1" >
<table>
<thead>
<tr>
<td> </td>
<th> <p class = "vertical_head center_head"></p> </th>
"""

# First row with samples
page_html+="""
<td><div class="span-1"><p class="rotation" style="alt"><b>Summary</b></p></div></td>"""
<th> <p class = "vertical_head center_head">Summary</p></th>"""

sorted_samples=sorted(dir_dict.keys())
for sample in sorted_samples:
sample_nick=sample
## For runs: put only the number after the _
#if "_" in sample:
#run_number=sample.split("_")[-1]
#if (not run_number.isalpha()) and len(run_number)>=6:
#sample_nick=run_number

if "_" in sample and "Data" not in sample:
sample_nick="_".join(sample.split("X_")[0].split("_")[:-1])
# Cleaning for MC: just the fragment
elif "Data" in sample and "RelVal" in sample:
sample_nick = "".join([sample.split("_")[0],sample.split("RelVal")[-1]])
# Cleaning for Data: PD + Era + Run
else:
sample_nick = sample

page_html+="""
<td><div class="span-1"><p class="rotation" style="">%s</p></div></td>"""%sample_nick
page_html+=" </tr>\n"
<th> <p class = "vertical_head center_head">%s</th></p>"""%sample_nick
page_html+="</tr> \n </thead> \n </tbody> \n"


# FIRST ROW
# Now the summaries at the beginning of the table
page_html+="<tr>"
page_html+='<td style="background-color:white;"><div class="span-1">'
page_html+='<td class="sticky-col">'

page_html+='<b>Summary</b></div></td>'
page_html+='<td style="background-color:white;" class = "colborder" ><div class="span-1"><img src="%s" alt="%s"></div></td>'%(global_dir.get_summary_chart_ajax(55,55),get_pie_tooltip(global_dir))
page_html+='<b>Summary</b></td>'
page_html+='<td><div class="span-1"> %s </div></td>'%(global_dir.get_piechart_js(50))
for sample in sorted_samples:
col=dir_dict[sample]
# check if the directory was a top one or not
summary_page_name="RelMonSummary.html"
if col.name!="":
summary_page_name=hash_name(col.name, hashing_flag)+".html"
img_link=col.get_summary_chart_ajax(55,55)
page_html+='<td style="background-color:white;"><div class="span-1">'
page_html+='<a href="%s/%s"><img src="%s" title="%s"></a></div></td>' %(sample,summary_page_name,img_link,get_pie_tooltip(col))
title = get_pie_tooltip(col)
chart = col.get_piechart_js(50,sample+"/"+summary_page_name)
page_html+='<td>'
page_html+='%s </a></td>' %(chart)
page_html+="</tr>"

# Now the content
for subdir_name in all_subdirs:

page_html+=' <tr>\n'
page_html+=' <td style="background-color:white;">%s</td>\n' %subdir_name
page_html+=' <td class="sticky-col" style="font-weight:bold;">%s</td>\n' %subdir_name

row_summary=Directory("row_summary","")
sample_counter=0
Expand All @@ -878,9 +905,10 @@ def make_summary_table(indir,aggregation_rules,aggregation_rules_twiki, hashing_

# one first row for the summary!
row_summary.calcStats()
img_link=row_summary.get_summary_chart_ajax(55,55)
page_html+='<td style="background-color:white;"><div class="span-1">'
page_html+='<img src="%s" title="%s"></div></td>' %(img_link,get_pie_tooltip(row_summary))
title = get_pie_tooltip(col)
chart = row_summary.get_piechart_js(50)
page_html+='<td><div>'#<div class="span-1">'
page_html+= chart + '</div></td>'

for sample in sorted_samples:
sample_counter+=1
Expand All @@ -903,30 +931,27 @@ def make_summary_table(indir,aggregation_rules,aggregation_rules_twiki, hashing_
summary_page=join(sample,"%s.html"%(hash_name(directory.name+subdir_name,hashing_flag)))
dir_is_there=subdir_name in subdirs_dict

img_link="https://chart.googleapis.com/chart?cht=p3&chco=C0C0C0&chs=50x50&chd=t:1"
img_tooltip="N/A"
if dir_is_there:
#row_summary.subdirs.append(subdirs_dict[subdir_name])
img_link=subdirs_dict[subdir_name].get_summary_chart_ajax(50,50)
img_tooltip=get_pie_tooltip(subdirs_dict[subdir_name])
img_link="https://upload.wikimedia.org/wikipedia/commons/a/a8/Circle_Davys-Grey_Solid.svg"
page_html+='<td><div class="span-1">'

page_html+='<td style="background-color:white;"><div class="span-1">'
if dir_is_there:
page_html+='<a href="%s">'%(summary_page)
page_html+='<img src="%s" title="%s" height=50 width=50>' %(img_link,img_tooltip)
if dir_is_there:
page_html+='</a>'
#row_summary.subdirs.append(subdirs_dict[subdir_name])
chart=subdirs_dict[subdir_name].get_piechart_js(50,summary_page)
page_html+='%s'%chart
else:
page_html+='<img src="%s" title="%s" height=50 width=50>' %(img_link,"Unavailable")
page_html+='</div></td>'

page_html+=" </tr>\n"



page_html+='</table> <a href="#top">Top...</a><hr>'
page_html+='</tbody> </table> </div> <a href="#top">Top...</a><hr>'

page_html+=get_rank_section(global_dir)

page_html+=make_twiki_table(dir_dict,aggregation_rules_twiki)
#page_html+=make_twiki_table(dir_dict,aggregation_rules_twiki)
# ^ commenting out for the moment, not really useful nor used

page_html+=get_page_footer()
return page_html
Expand Down
47 changes: 37 additions & 10 deletions Utilities/RelMon/python/dirstructure.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
from copy import deepcopy
from os import chdir,getcwd,listdir,makedirs,rmdir
from os.path import exists,join
import random

import sys
argv=sys.argv
Expand Down Expand Up @@ -167,19 +168,45 @@ def get_subdirs_names(self):
subdirnames.append(subdir.name)
return subdirnames

def get_summary_chart_ajax(self,w=400,h=300):
"""Emit the ajax to build a pie chart using google apis...
def get_piechart_js(self,w=400,link=None):

"""
Build the HTML snippet to render a piechart with chart.js
"""
url = "https://chart.googleapis.com/chart?"
url+= "cht=p3" # Select the 3d chart
#url+= "&chl=Success|Null|Fail" # give labels
url+= "&chco=00FF00|FFFF00|FF0000|7A7A7A" # give colours to labels
url+= "&chs=%sx%s" %(w,h)
#url+= "&chtt=%s" %self.name
url+= "&chd=t:%.2f,%.2f,%.2f,%.2f"%(self.get_success_rate(),self.get_null_rate(),self.get_fail_rate(),self.get_skiped_rate())
if self.get_success_rate()>=99.9: # if the success rate is very high let's make the page lighter
img_link = "https://raw.githubusercontent.com/cms-PdmV/RelMonService2/5ee98db210c0898fd34b4deac3653fa2bdff269b/report_website/lime_circle.png"
html ='<img src="%s" height=%d width=%d>' %(img_link,w,w)
if link is not None:
html = '<a href="%s"> %s </a>' %(link,html)
return html

name = random.getrandbits(64) # just a random has for the canvas
html = ""
html += '<canvas id="%s" height=%d width=%d></canvas>'%(name,w,w)
# piechart
html += '<script> new Chart("%s",'%(name)
html += '{ type: "pie",'

# data
html += 'data: {'
html += 'labels: ["Success", "Null" , "Failure", "Skipped"],'
html += 'datasets: [{ backgroundColor: ["lime","yellow","red","grey"],'
html += 'data: [%.2f,%.2f,%.2f,%.2f]}] },'%(self.get_success_rate(),self.get_null_rate(),self.get_fail_rate(),self.get_skiped_rate())

return url
#display options
html += 'options: { '

if link is not None:
html += 'onClick : function(event) { window.open("%s", "_blank");},'%(link)


html +='legend: { display: false }, responsive : false, hover: {mode: null}, tooltips: {enabled: false}'
#tooltips: {enabled: false}, hover: {mode: null},'

html += '}}); </script>'

return html

def print_report(self,indent="",verbose=False):
if len(indent)==0:
self.calcStats(make_pie=False)
Expand Down