-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathsummary.ts
171 lines (145 loc) · 7.55 KB
/
summary.ts
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
import { Scale } from "./scale";
export class Summary {
private static toFixed(input: number) {
let roundedString: string = input.toFixed(2); // This will be "24.44" as a string
// Convert the string back to a number using the unary plus operator
let roundedNumber: number = +roundedString;
return roundedNumber;
}
public static calculateOptionCount(data: any[]): any {
const ervy = require('ervy')
const { fg } = ervy
let styles = [fg('cyan', '+ '), fg('magenta', '- '), fg('yellow', '0 '), fg('red', '* '), fg('green', 'x '), fg('blue', 'o '), fg('white', '# ')]
const optionValuesCount: any = {};
const optionValuesArray = data.map(group => {
const optionValues = group.map((entry: any) => JSON.parse(entry.result).optionValue);
// count instances of each optionValues
return optionValues;
});
// sum up appearances of each optionValue inside optionValues
for (let optionValues of optionValuesArray) {
for (let optionValue of optionValues) {
if (optionValuesCount[optionValue]) {
optionValuesCount[optionValue]++;
} else {
optionValuesCount[optionValue] = 1;
}
}
}
let graphResults = [];
let i = 0;
for (let optionValue in optionValuesCount) {
// TODO: make sure there enough styles for all the optionValues
graphResults.push({ key: optionValue, value: optionValuesCount[optionValue], style: styles[i] });
i++;
}
return graphResults;
}
public static calculateAverageByModel(data: any[]): any {
const results = data.map(group => {
// Assuming all entries in a group are from the same model
const ervy = require('ervy')
const { bg } = ervy
const modelName = group[0].model;
// Extract probabilities and parse them
const probabilities = group.map((entry: any) => JSON.parse(entry.result).probability);
// Calculate average
const avg = probabilities.filter((prob: any) => prob != "N/A").reduce((acc: any, curr: any) => acc + curr, 0) / probabilities.length;
return { key: modelName, value: this.toFixed(avg) || 0, style: bg('blue') };
});
return results;
}
public static calculateAverageConfidenceLevelByModel(data: any[]): any {
const results = data.map(group => {
// Assuming all entries in a group are from the same model
const ervy = require('ervy')
const { bg } = ervy
const modelName = group[0].model;
// Extract probabilities and parse them
const confidenceLevels = group.map((entry: any) => JSON.parse(entry.result).confidenceLevel);
// Calculate average
const avg = confidenceLevels.filter((prob: any) => prob != "N/A").reduce((acc: any, curr: any) => acc + curr, 0) / confidenceLevels.length;
return { key: modelName, value: this.toFixed(avg) || 0, style: bg('red') };
});
return results;
}
public static listReasoning(data: any[]): any {
const results = data.map(group => {
// Assuming all entries in a group are from the same model
const modelName = group[0].model;
const reasoning = group.map((entry: any) => JSON.parse(entry.result)?.reasoning);
return { key: modelName, value: reasoning };
});
return results;
}
public static calculateStandardDeviationByModel(data: any[]): any {
const results = data.map(group => {
// Assuming all entries in a group are from the same model
const modelName = group[0].model;
// Extract probabilities and parse them
const probabilities = group.map((entry: any) => JSON.parse(entry.result).probability);
// Calculate average
const avg = probabilities.filter((prob: any) => prob != "N/A").reduce((acc: any, curr: any) => acc + curr, 0) / probabilities.length;
// Calculate variance
const variance = probabilities.reduce((acc: any, curr: any) => acc + Math.pow(curr - avg, 2), 0) / probabilities.length;
let standardDeviation = Math.sqrt(variance);
const ervy = require('ervy')
const { bg } = ervy
return { key: modelName, value: this.toFixed(standardDeviation) || 0, style: bg('blue') };
});
return results;
}
public static calculateTotals(data: any[]): any {
// Flatten all probabilities into a single array
const allProbabilities = data.flatMap(group =>
group.map((entry: any) => JSON.parse(entry.result).probability)
);
// Calculate total average
const totalAvg = allProbabilities.filter((prob: any) => prob != "N/A").reduce((acc, curr) => acc + curr, 0) / allProbabilities.length;
// Calculate total variance
const totalVariance = allProbabilities.reduce((acc, curr) =>
acc + Math.pow(curr - totalAvg, 2), 0) / allProbabilities.length;
// Calculate total standard deviation
const totalStandardDeviation = Math.sqrt(totalVariance);
return { totalAverage: this.toFixed(totalAvg), totalVariance: this.toFixed(totalVariance), totalStandardDeviation: this.toFixed(totalStandardDeviation) };
};
public static create(data: any[], scale: Scale, verbose: boolean = false): void {
const ervy = require('ervy')
const { bar, pie, bullet, donut, gauge, scatter } = ervy
switch (scale) {
case Scale.Probability:
{
let barAverageData = this.calculateAverageByModel(data);
let barStandardDeviationData = this.calculateStandardDeviationByModel(data)
let barAverageConfidenceLevelData = this.calculateAverageConfidenceLevelByModel(data)
console.log("Average by Model: ");
console.log(bar(barAverageData, { barWidth: 20 }));
console.log("Standard Deviation by Model: ");
console.log(bar(barStandardDeviationData, { barWidth: 20 }));
console.log("Average Confidence Level by Model: ");
console.log(bar(barAverageConfidenceLevelData, { barWidth: 20 }));
let totals = this.calculateTotals(data);
console.log("Average: " + totals.totalAverage + " Variance: " + totals.totalVariance + " Standard Deviation: " + totals.totalStandardDeviation);
if (verbose) {
console.log("Reasoning: ");
console.log(this.listReasoning(data));
}
}
break;
case Scale.Options:
// here there should be graphs by model
// each graph should present the options
console.log("Options Count: ");
let countByModel = this.calculateOptionCount(data);
console.log(donut(countByModel));
console.log("Average Confidence Level by Model: ");
let barAverageConfidenceLevelData = this.calculateAverageConfidenceLevelByModel(data)
console.log(bar(barAverageConfidenceLevelData, { barWidth: 20 }));
if (verbose) {
console.log("Reasoning: ");
console.log(this.listReasoning(data));
}
default:
}
}
}