Skip to content

Commit

Permalink
Merge remote-tracking branch 'usnistgov/fix/metrics-crash' into test/…
Browse files Browse the repository at this point in the history 
…metrics2023
  • Loading branch information
@RayPlante
RayPlante committed Jan 9, 2024
2 parents 2ff466f + d9e6b02 commit a7e0e54
Show file tree
Hide file tree
Showing 7 changed files with 138 additions and 94 deletions.
2 changes: 1 addition & 1 deletion angular/src/app/app.component.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ export class AppComponent {
this.gaCode = this.cfg.get("gaCode", "") as string;
this.ga4Code = this.cfg.get("ga4Code", "") as string;
let homeurl = this.cfg.get("locations.portalBase", "data.nist.gov") as string;
console.log('homeurl', homeurl);

const url = new URL(homeurl);
this.hostName = url.hostname;

Expand Down
14 changes: 11 additions & 3 deletions angular/src/app/landing/data-files/data-files.component.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -273,7 +273,7 @@ export class DataFilesComponent implements OnInit, OnChanges {
}

this.buildTree();
console.log("files", this.files);

// If total file count > virtual scrolling threshold, set virtual scrolling to true.
this.virtualScroll = this.fileCount > FileCountForVirtualScroll? true : false;

Expand All @@ -282,7 +282,14 @@ export class DataFilesComponent implements OnInit, OnChanges {
}

isRestrictedData(node: any) {
return node['comp']['@type'].includes('nrdp:RestrictedAccessPage');
if(node){
if(node['comp'])
return node['comp']['@type'].includes('nrdp:RestrictedAccessPage');
else
return false;
}else{
return false;
}
}

checkAccessPageType() {
Expand Down Expand Up @@ -631,7 +638,8 @@ export class DataFilesComponent implements OnInit, OnChanges {
* @returns boolean
*/
isLeaf(fileNode: any) {
return (fileNode.comp['@type'].indexOf('nrdp:DataFile') > -1);
if(!fileNode.comp) return false;
else return (fileNode.comp['@type'].indexOf('nrdp:DataFile') > -1);
}

/**
Expand Down
11 changes: 4 additions & 7 deletions angular/src/app/landing/landingpage.component.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -250,9 +250,6 @@ export class LandingPageComponent implements OnInit, AfterViewInit {

if (this.editRequested) {
showError = false;
// console.log("Returning from authentication redirection (editmode="+
// this.editRequested+")");

// Need to pass reqID (resID) because the resID in editControlComponent
// has not been set yet and the startEditing function relies on it.
this.edstatsvc.startEditing(this.reqId);
Expand Down Expand Up @@ -304,12 +301,12 @@ export class LandingPageComponent implements OnInit, AfterViewInit {
let hasFile = false;

if(this.md.components && this.md.components.length > 0){
this.md.components.forEach(element => {
if(element.filepath){
for(let com of this.md.components) {
if(com.filepath){
hasFile = true;
return;
break;
}
});
}
}

if(hasFile){
Expand Down
10 changes: 0 additions & 10 deletions angular/src/app/metrics/horizontal-barchart/horizontal-barchart.component.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -294,16 +294,6 @@ export class HorizontalBarchartComponent implements OnInit {
.attr('opacity', 0.6)
.style('fill', '#00e68a')

// draw a yellow dash line at the end of the active bar
// line style is defined in line#limit
chart.append('line')
.attr('id', 'limit')
.attr('x1', xScale(actual[1]))
.attr('y1', 0)
.attr('x2', xScale(actual[1]))
.attr('y2', height)
.attr('stroke', 'red')

d3.selectAll('.label')
.filter(function(d) { return d[0] == actual[0]; })
.attr('opacity', 1)
Expand Down
6 changes: 3 additions & 3 deletions angular/src/app/metrics/metrics.component.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@
<div style="clear: both;"></div>
<div class="min-height-20">
<span class="float-left">Total dataset downloads</span>
<span class="float-right">{{TotalDatasetDownloads}}</span>
<span class="float-right">{{totalDatasetDownloads}}</span>
</div>
<div style="clear: both;"></div>
<div class="min-height-20">
Expand All @@ -62,7 +62,7 @@
<div style="clear: both;"></div>
<div class="min-height-20">
<span class="float-left">Total unique users</span>
<span class="float-right">{{TotalUniqueUsers}}</span>
<span class="float-right">{{totalUniqueUsers}}</span>
</div>
<div style="clear: both;"></div>
<div class="min-height-20">
Expand Down Expand Up @@ -149,7 +149,7 @@
</div>

<!-- Display file level data summary -->
<div *ngIf="filescount > 0" style="margin: 1em auto 0em auto; width: 100%; text-align: right;font-size: small;padding-right: 45px;">Total No. files: {{filescount}}, Total dataset size: {{TotalFileSize}}</div>
<div *ngIf="filescount > 0" style="margin: 1em auto 0em auto; width: 100%; text-align: right;font-size: small;padding-right: 45px;">Total No. files: {{filescount}}, Total dataset size: {{totalFileSizeForDisplay}}</div>

<!-- Display tree table -->
<div #panel0 id="panel0" *ngIf="inBrowser; else filesLoading" class="flex-container" >
Expand Down
136 changes: 68 additions & 68 deletions angular/src/app/metrics/metrics.component.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ export class MetricsComponent implements OnInit {

// Data
ediid: string;
pdrid: string;
files: TreeNode[] = [];
fileLevelData: any;
firstTimeLogged: string = '';
Expand All @@ -43,8 +44,12 @@ export class MetricsComponent implements OnInit {
metricsData: any[] = [];
totalFileLevelSuccessfulGet: number = 0;
totalFileSize: number = 0;
totalFileSizeForDisplay: string = "";
totalFilesinChart: number = 0;
noDatasetSummary: boolean = false;
totalDatasetDownloads: number = 0;
totalUniqueUsers: number = 0;
totalDownloadSizeInByte: number = 0;

// Chart
chartData: Array<any>;
Expand Down Expand Up @@ -117,6 +122,7 @@ export class MetricsComponent implements OnInit {
if(md) {
this.record = md as NerdmRes;
this.datasetTitle = md['title'];
this.pdrid = md['@id'];

this.createNewDataHierarchy();
if (this.files.length != 0){
Expand All @@ -136,7 +142,6 @@ export class MetricsComponent implements OnInit {
this.cleanupFileLevelData(this.files);
this.fileLevelData.FilesMetrics = this.metricsData;
this.handleSum(this.files);

if(this.fileLevelData.FilesMetrics.length > 0){
this.noChartData = false;
this.createChartData();
Expand Down Expand Up @@ -182,9 +187,8 @@ export class MetricsComponent implements OnInit {
this.recordLevelData = JSON.parse(await event.body.text());

if(this.recordLevelData.DataSetMetrics != undefined && this.recordLevelData.DataSetMetrics.length > 0){
this.firstTimeLogged = this.datePipe.transform(this.recordLevelData.DataSetMetrics[0].first_time_logged, "MMM d, y");

this.recordLevelTotalDownloads = this.recordLevelData.DataSetMetrics[0].success_get;
this.handleRecordLevelData();

// this.xAxisLabel = "Total Downloads Since " + this.firstTimeLogged;
this.datasetSubtitle = "Metrics Since " + this.firstTimeLogged;
Expand Down Expand Up @@ -245,38 +249,50 @@ export class MetricsComponent implements OnInit {
}, 0);
}


handleRecordLevelData() {
// this.recordLevelTotalDownloads = this.recordLevelData.DataSetMetrics[0].success_get;

for(let metrics of this.recordLevelData.DataSetMetrics) {
if(!this.pdrid || metrics["pdrid"].toLowerCase() == 'nan' || metrics["pdrid"].trim() == this.pdrid){
this.firstTimeLogged = this.datePipe.transform(metrics.first_time_logged, "MMM d, y");
this.recordLevelTotalDownloads = metrics.success_get;
this.totalDatasetDownloads = metrics.record_download;
this.totalUniqueUsers = metrics.number_users;
this.totalDownloadSizeInByte = metrics["total_size_download"];
}
}
}

/**
* Remove outdated data and sha files from the metrics
*/
cleanupFileLevelData(files: TreeNode[]){
let metricsData: any[] = [];

for(let node of files){
// let found = this.fileLevelData.FilesMetrics.find(x => x.filepath.substr(x.filepath.indexOf(this.ediid)+this.ediid.length).trim() == node.data.filePath.trim() && !x.filepath.endsWith('sha256'));

let filenameWithPath = node.data.filePath[0] == '/' ? node.data.filePath.substr(1): node.data.filePath.trim();

let found = this.fileLevelData.FilesMetrics.find(x => x.filepath.trim() == filenameWithPath && !x.filepath.endsWith('sha256'));
if(found){
metricsData.push(found);
node.data.success_get = found.success_get;
if(!node.data.download_size || node.data.download_size == 0){
node.data.download_size = found.download_size;
}
//Only check leaf
if(node.children.length <= 0) {
let found = this.metricsService.findFileLevelMatch(this.fileLevelData.FilesMetrics, node.data.ediid, node.data.pdrid, node.data.filePath);

if(found){
metricsData.push(found);
node.data.success_get = found.success_get;
if(!node.data.download_size || node.data.download_size == 0){
node.data.download_size = found.download_size;
}

this.totalFileLevelSuccessfulGet += found.success_get;
this.totalFilesinChart += 1;
this.totalFileLevelSuccessfulGet += found.success_get;
this.totalFilesinChart += 1;

node.data.inChart = true;
if(node.parent){
node.parent.data.inChart = true;
node.data.inChart = true;
if(node.parent){
node.parent.data.inChart = true;
}
}
}

if(node.children.length > 0){
this.cleanupFileLevelData(node.children);
}else{
this.filescount = this.filescount + 1;
}else {
this.cleanupFileLevelData(node.children);
}
}

Expand All @@ -297,6 +313,8 @@ export class MetricsComponent implements OnInit {
const {downloads, fileSize} = this.sumFolder(child);
this.totalFileSize += fileSize;
}

this.totalFileSizeForDisplay = this.commonFunctionService.formatBytes(this.totalFileSize, 2);
}

/**
Expand All @@ -314,7 +332,13 @@ export class MetricsComponent implements OnInit {
}

var downloads = node.data.success_get;
var fileSize = node.data.download_size;

var fileSize;
if(!node.data.download_size || node.data.download_size == 'nan')
fileSize = 0;
else
fileSize = node.data.download_size;

return {downloads, fileSize};
}

Expand Down Expand Up @@ -347,9 +371,9 @@ export class MetricsComponent implements OnInit {
// Add summary
csv = "# Record id," + this.ediid + "\r\n"
+ "# Total file downloads," + this.recordLevelTotalDownloads + "\r\n"
+ "# Total dataset downloads," + this.TotalDatasetDownloads + "\r\n"
+ "# Total dataset downloads," + this.totalDatasetDownloads + "\r\n"
+ "# Total bytes downloaded," + this.totalDownloadSizeInByte + "\r\n"
+ "# Total unique users," + this.TotalUniqueUsers + "\r\n"
+ "# Total unique users," + this.totalUniqueUsers + "\r\n"
+ "\r\n" + csv;

// Create link and download
Expand All @@ -362,34 +386,12 @@ export class MetricsComponent implements OnInit {
document.body.removeChild(link);
}

/**
* Return total dataset downloads
*/
get TotalDatasetDownloads() {
if(this.recordLevelData.DataSetMetrics[0] != undefined){
return this.recordLevelData.DataSetMetrics[0].record_download;
}else{
return ""
}
}

/**
* Return total download size from file level summary
*/
get TotalFileSize() {
return this.commonFunctionService.formatBytes(this.totalFileSize, 2);
}

/**
* Get total unique users
*/
get TotalUniqueUsers() {
if(this.recordLevelData.DataSetMetrics[0] != undefined){
return this.recordLevelData.DataSetMetrics[0].number_users;
}else{
return ""
}
}
// get TotalFileSize() {
// return this.commonFunctionService.formatBytes(this.totalFileSize, 2);
// }

/**
* Save the bar chart as a png file
Expand Down Expand Up @@ -499,9 +501,9 @@ export class MetricsComponent implements OnInit {
*/
getLastDownloadDate(){
if (this.fileLevelData.FilesMetrics.length) {
var lastDownloadTime = this.fileLevelData.FilesMetrics.reduce((m,v,i) => (v.timestamp > m.timestamp) && i ? v : m).timestamp;
var lastDownloadTime = this.fileLevelData.FilesMetrics.reduce((m,v,i) => (v.last_time_logged > m.last_time_logged) && i ? v : m).last_time_logged;

return this.datePipe.transform(this.fileLevelData.FilesMetrics.reduce((m,v,i) => (v.timestamp > m.timestamp) && i ? v : m).timestamp, "MMM d, y");
return this.datePipe.transform(this.fileLevelData.FilesMetrics.reduce((m,v,i) => (v.last_time_logged > m.last_time_logged) && i ? v : m).last_time_logged, "MMM d, y");
}
}

Expand All @@ -519,21 +521,18 @@ export class MetricsComponent implements OnInit {

/**
* Get total recordset level download size in bytes
* 01/08/2024 Discussed with Deoyani, use "total_size_download" in record level data.
* No need to add file level data anymore.
*/
get totalDownloadSizeInByte() {
let totalDownload = 0;
if(this.fileLevelData != undefined) {
for(let file of this.fileLevelData.FilesMetrics) {
totalDownload += file.success_get * file.download_size;
};
}
// get totalDownloadSizeInByte() {
// let totalDownload = 0;

if(this.recordLevelData != undefined) {
totalDownload += this.recordLevelData.DataSetMetrics[0]["total_size_download"];
}
// if(this.recordLevelData != undefined) {
// totalDownload += this.recordLevelData.DataSetMetrics[0]["total_size_download"];
// }

return totalDownload;
}
// return totalDownload;
// }

/**
* Reture style for Title column of the file tree
Expand Down Expand Up @@ -605,7 +604,7 @@ export class MetricsComponent implements OnInit {
createNewDataHierarchy() {
var testdata: TreeNode = {}
if (this.record['components'] != null) {
testdata["data"] = this.arrangeIntoTree(this.record['components']);
testdata["data"] = this.arrangeIntoTree(this.record['components'], this.record['@id']);
this.files.push(testdata);
}
}
Expand All @@ -615,7 +614,7 @@ export class MetricsComponent implements OnInit {
* @param paths
* @returns
*/
private arrangeIntoTree(paths) {
private arrangeIntoTree(paths, pdrid) {
const tree: TreeNode[] = [];
// This example uses the underscore.js library.
var i = 1;
Expand Down Expand Up @@ -648,6 +647,7 @@ export class MetricsComponent implements OnInit {
data: {
cartId: tempId,
ediid: this.ediid,
pdrid: pdrid,
name: part,
mediatype: path.mediaType,
size: path.size,
Expand Down
Loading

0 comments on commit a7e0e54

Please sign in to comment.