Skip to content

Commit

Permalink
replaced shap with deeplift --> too computationaly expensive
Browse files Browse the repository at this point in the history
  • Loading branch information
@GChrysostomou
GChrysostomou committed Aug 7, 2021
1 parent 4eb0192 commit 26e2a7f
Show file tree
Hide file tree
Showing 28 changed files with 1,140 additions and 922 deletions.
20 changes: 12 additions & 8 deletions double_faithfulness_metrics/sst/contigious-dev-f1-metrics-description.json
View file
Original file line number Diff line number Diff line change
@@ -1,38 +1,42 @@
{
"f1 macro avg - model labels": {
"fixed-fixed-len_var-feat_var-type": 37.302,
"fixed-shap": 79.643,
"fixed-lime": 73.755,
"fixed-attention": 80.734,
"fixed-ig": 70.803,
"fixed-gradients": 78.178,
"fixed-scaled attention": 62.218,
"fixed-random": 82.901,
"fixed-fixed-len_var-feat": 60.48,
"var-var-len_var-feat_var-type": 36.246,
"fixed-fixed-len_var-feat": 61.049,
"fixed-fixed-len_var-feat_var-type": 36.3,
"var-shap": 80.025,
"var-lime": 72.084,
"var-attention": 83.397,
"var-ig": 66.115,
"var-gradients": 75.114,
"var-scaled attention": 54.928,
"var-random": 76.832,
"var-var-len_var-feat": 52.282
"var-var-len_var-feat": 51.574,
"var-var-len_var-feat_var-type": 35.118
},
"f1 macro avg - actual labels": {
"fixed-fixed-len_var-feat_var-type": 42.749,
"fixed-shap": 77.903,
"fixed-lime": 72.847,
"fixed-attention": 77.638,
"fixed-ig": 71.815,
"fixed-gradients": 76.214,
"fixed-scaled attention": 64.378,
"fixed-random": 80.255,
"fixed-fixed-len_var-feat": 62.174,
"var-var-len_var-feat_var-type": 42.129,
"fixed-fixed-len_var-feat": 63.207,
"fixed-fixed-len_var-feat_var-type": 41.509,
"var-shap": 78.984,
"var-lime": 71.643,
"var-attention": 79.608,
"var-ig": 69.201,
"var-gradients": 75.229,
"var-scaled attention": 59.855,
"var-random": 77.402,
"var-var-len_var-feat": 56.74
"var-var-len_var-feat": 56.032,
"var-var-len_var-feat_var-type": 40.996
}
}
104 changes: 62 additions & 42 deletions double_faithfulness_metrics/sst/contigious-dev-faithfulness-metrics-description.json
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
{
"fixed-fixed-len_var-feat_var-type": {
"fixed-shap": {
"sufficiency": {
"mean": 0.8007193730306734,
"std": 0.3390950525499532
"mean": 0.610242849839595,
"std": 0.4334832476420736
},
"comprehensiveness": {
"mean": 0.6856246412304128,
"std": 0.4204862792130052
"mean": 0.3432124525969136,
"std": 0.41816196873160466
}
},
"fixed-lime": {
Expand Down Expand Up @@ -71,92 +71,112 @@
},
"fixed-fixed-len_var-feat": {
"sufficiency": {
"mean": 0.7779232331020534,
"std": 0.3628206280268484
"mean": 0.7744772575426913,
"std": 0.36637767820355216
},
"comprehensiveness": {
"mean": 0.5552632009514311,
"std": 0.4403901974505537
"mean": 0.5527675434413967,
"std": 0.44028594708915886
}
},
"var-var-len_var-feat_var-type": {
"fixed-fixed-len_var-feat_var-type": {
"sufficiency": {
"mean": 0.7962548369840813,
"std": 0.3426570809503985
},
"comprehensiveness": {
"mean": 0.6887664696021752,
"std": 0.41797846058640975
}
},
"var-shap": {
"sufficiency": {
"mean": 0.7258237360147844,
"std": 0.3867909363515245
"mean": 0.5877084732147781,
"std": 0.4310697893813187
},
"comprehensiveness": {
"mean": 0.7047298798050412,
"std": 0.42143334583635933
"mean": 0.3500142024218937,
"std": 0.4244923942117259
}
},
"var-lime": {
"sufficiency": {
"mean": 0.5853310176886943,
"std": 0.40228381873402874
"mean": 0.5854836281484053,
"std": 0.40226005340701826
},
"comprehensiveness": {
"mean": 0.4095809419306717,
"std": 0.4655954837501286
"mean": 0.4094833984725633,
"std": 0.4655734775924548
}
},
"var-attention": {
"sufficiency": {
"mean": 0.5350261610353776,
"std": 0.4466267000194526
"mean": 0.5351186686632258,
"std": 0.4466166165367489
},
"comprehensiveness": {
"mean": 0.2894682585221177,
"std": 0.404836114720524
"mean": 0.2893412216476672,
"std": 0.40477317699040327
}
},
"var-ig": {
"sufficiency": {
"mean": 0.6784694899565651,
"std": 0.41089852525844817
"mean": 0.678588370907915,
"std": 0.41085893750762276
},
"comprehensiveness": {
"mean": 0.4919090986305166,
"std": 0.4487749089765129
"mean": 0.4917824611223237,
"std": 0.44876830646829
}
},
"var-gradients": {
"sufficiency": {
"mean": 0.5955711199925428,
"std": 0.43198743319702937
"mean": 0.5957053986476406,
"std": 0.43194617255574697
},
"comprehensiveness": {
"mean": 0.40171962165298225,
"std": 0.4391584324008698
"mean": 0.401571529695811,
"std": 0.4391112799345842
}
},
"var-scaled attention": {
"sufficiency": {
"mean": 0.7715550531737865,
"std": 0.37202796683090217
"mean": 0.7716881577283307,
"std": 0.37192823800824837
},
"comprehensiveness": {
"mean": 0.613632415534318,
"std": 0.425138935996291
"mean": 0.6135021407663697,
"std": 0.42515866882504416
}
},
"var-random": {
"sufficiency": {
"mean": 0.579835489194161,
"std": 0.4425401755097988
"mean": 0.5799295994978603,
"std": 0.44252856149403563
},
"comprehensiveness": {
"mean": 0.36924572728434496,
"std": 0.42835132857130515
"mean": 0.3691073639212445,
"std": 0.4283157686423889
}
},
"var-var-len_var-feat": {
"sufficiency": {
"mean": 0.7286819708875355,
"std": 0.38511129667292576
"mean": 0.7296611861958074,
"std": 0.3850971713456964
},
"comprehensiveness": {
"mean": 0.5996638883377581,
"std": 0.44810445282384803
}
},
"var-var-len_var-feat_var-type": {
"sufficiency": {
"mean": 0.7292880665493804,
"std": 0.3839620687648042
},
"comprehensiveness": {
"mean": 0.5994067736367245,
"std": 0.44791596543842294
"mean": 0.7090057531647765,
"std": 0.4195702552154092
}
}
}
20 changes: 12 additions & 8 deletions double_faithfulness_metrics/sst/contigious-test-f1-metrics-description.json
View file
Original file line number Diff line number Diff line change
@@ -1,38 +1,42 @@
{
"f1 macro avg - model labels": {
"fixed-fixed-len_var-feat_var-type": 37.774,
"fixed-shap": 80.385,
"fixed-lime": 70.953,
"fixed-attention": 82.14,
"fixed-ig": 71.027,
"fixed-gradients": 79.156,
"fixed-scaled attention": 59.785,
"fixed-random": 81.656,
"fixed-fixed-len_var-feat": 58.537,
"var-var-len_var-feat_var-type": 36.17,
"fixed-fixed-len_var-feat": 59.646,
"fixed-fixed-len_var-feat_var-type": 37.291,
"var-shap": 78.929,
"var-lime": 69.849,
"var-attention": 84.89,
"var-ig": 65.555,
"var-gradients": 75.328,
"var-scaled attention": 54.453,
"var-random": 77.55,
"var-var-len_var-feat": 51.52
"var-var-len_var-feat": 51.128,
"var-var-len_var-feat_var-type": 35.415
},
"f1 macro avg - actual labels": {
"fixed-fixed-len_var-feat_var-type": 43.137,
"fixed-shap": 77.745,
"fixed-lime": 71.349,
"fixed-attention": 79.093,
"fixed-ig": 71.982,
"fixed-gradients": 77.39,
"fixed-scaled attention": 63.476,
"fixed-random": 80.456,
"fixed-fixed-len_var-feat": 61.531,
"var-var-len_var-feat_var-type": 42.199,
"fixed-fixed-len_var-feat": 61.972,
"fixed-fixed-len_var-feat_var-type": 42.069,
"var-shap": 78.396,
"var-lime": 70.814,
"var-attention": 80.889,
"var-ig": 68.504,
"var-gradients": 76.214,
"var-scaled attention": 59.489,
"var-random": 78.337,
"var-var-len_var-feat": 56.514
"var-var-len_var-feat": 56.007,
"var-var-len_var-feat_var-type": 41.211
}
}
Loading

0 comments on commit 26e2a7f

Please sign in to comment.