running also other experiments

double_jsd/faithfulness_metrics/agnews/contigious-dev-f1-metrics-description.json

@@ -0,0 +1,42 @@
+{
+    "f1 macro avg - model labels": {
+        "fixed-deeplift": 89.106,
+        "fixed-lime": 92.449,
+        "fixed-attention": 86.005,
+        "fixed-ig": 83.144,
+        "fixed-gradients": 89.246,
+        "fixed-scaled attention": 81.387,
+        "fixed-random": 89.825,
+        "fixed-fixed-len_var-feat": 78.879,
+        "fixed-fixed-len_var-feat_var-type": 39.505,
+        "var-deeplift": 85.558,
+        "var-lime": 88.934,
+        "var-attention": 84.675,
+        "var-ig": 80.164,
+        "var-gradients": 87.935,
+        "var-scaled attention": 79.42,
+        "var-random": 87.641,
+        "var-var-len_var-feat": 76.284,
+        "var-var-len_var-feat_var-type": 39.923
+    },
+    "f1 macro avg - actual labels": {
+        "fixed-deeplift": 86.761,
+        "fixed-lime": 89.474,
+        "fixed-attention": 83.446,
+        "fixed-ig": 81.891,
+        "fixed-gradients": 86.416,
+        "fixed-scaled attention": 80.657,
+        "fixed-random": 86.913,
+        "fixed-fixed-len_var-feat": 78.159,
+        "fixed-fixed-len_var-feat_var-type": 41.356,
+        "var-deeplift": 84.513,
+        "var-lime": 87.237,
+        "var-attention": 82.715,
+        "var-ig": 80.065,
+        "var-gradients": 85.876,
+        "var-scaled attention": 79.852,
+        "var-random": 85.77,
+        "var-var-len_var-feat": 76.51,
+        "var-var-len_var-feat_var-type": 42.062
+    }
+}

double_jsd/faithfulness_metrics/agnews/contigious-dev-faithfulness-metrics-description.json

@@ -0,0 +1,182 @@
+{
+    "fixed-deeplift": {
+        "sufficiency": {
+            "mean": 0.629022245962207,
+            "std": 0.40393264665376205
+        },
+        "comprehensiveness": {
+            "mean": 0.18782505961708434,
+            "std": 0.3445388384408546
+        }
+    },
+    "fixed-lime": {
+        "sufficiency": {
+            "mean": 0.5257975424393123,
+            "std": 0.4211294243792812
+        },
+        "comprehensiveness": {
+            "mean": 0.1766856035497435,
+            "std": 0.3448934825561432
+        }
+    },
+    "fixed-attention": {
+        "sufficiency": {
+            "mean": 0.6884252633228786,
+            "std": 0.38469386143140955
+        },
+        "comprehensiveness": {
+            "mean": 0.21072120294506858,
+            "std": 0.3608933502974825
+        }
+    },
+    "fixed-ig": {
+        "sufficiency": {
+            "mean": 0.7071405400493309,
+            "std": 0.3722865990725928
+        },
+        "comprehensiveness": {
+            "mean": 0.24470696202585188,
+            "std": 0.3812462258178099
+        }
+    },
+    "fixed-gradients": {
+        "sufficiency": {
+            "mean": 0.6252560670847443,
+            "std": 0.4047567526527378
+        },
+        "comprehensiveness": {
+            "mean": 0.1683110498662155,
+            "std": 0.3295896350131446
+        }
+    },
+    "fixed-scaled attention": {
+        "sufficiency": {
+            "mean": 0.717201884349895,
+            "std": 0.37156395856727686
+        },
+        "comprehensiveness": {
+            "mean": 0.2555325048543113,
+            "std": 0.3873784065451631
+        }
+    },
+    "fixed-random": {
+        "sufficiency": {
+            "mean": 0.6109224129181776,
+            "std": 0.41146263454803134
+        },
+        "comprehensiveness": {
+            "mean": 0.16006318458342877,
+            "std": 0.3190711139977896
+        }
+    },
+    "fixed-fixed-len_var-feat": {
+        "sufficiency": {
+            "mean": 0.7667281862005435,
+            "std": 0.3375289339172482
+        },
+        "comprehensiveness": {
+            "mean": 0.30882255559042726,
+            "std": 0.41280445637444846
+        }
+    },
+    "fixed-fixed-len_var-feat_var-type": {
+        "sufficiency": {
+            "mean": 0.8319384434170248,
+            "std": 0.3066147069892663
+        },
+        "comprehensiveness": {
+            "mean": 0.714503720997817,
+            "std": 0.4005860363938825
+        }
+    },
+    "var-deeplift": {
+        "sufficiency": {
+            "mean": 0.5548639019121792,
+            "std": 0.4148930242950225
+        },
+        "comprehensiveness": {
+            "mean": 0.24573084348269902,
+            "std": 0.38561487652756776
+        }
+    },
+    "var-lime": {
+        "sufficiency": {
+            "mean": 0.42035435298538976,
+            "std": 0.4145061506788717
+        },
+        "comprehensiveness": {
+            "mean": 0.22087906250495043,
+            "std": 0.38123578916625916
+        }
+    },
+    "var-attention": {
+        "sufficiency": {
+            "mean": 0.5665725843379797,
+            "std": 0.41869973196250554
+        },
+        "comprehensiveness": {
+            "mean": 0.21805978875586465,
+            "std": 0.37157218057374025
+        }
+    },
+    "var-ig": {
+        "sufficiency": {
+            "mean": 0.6292912859380662,
+            "std": 0.39921558957465775
+        },
+        "comprehensiveness": {
+            "mean": 0.2857363029098707,
+            "std": 0.4041715810416819
+        }
+    },
+    "var-gradients": {
+        "sufficiency": {
+            "mean": 0.5178077943366559,
+            "std": 0.4206257535072318
+        },
+        "comprehensiveness": {
+            "mean": 0.18050668988106255,
+            "std": 0.3437988415647928
+        }
+    },
+    "var-scaled attention": {
+        "sufficiency": {
+            "mean": 0.6232606310502106,
+            "std": 0.4080316704713507
+        },
+        "comprehensiveness": {
+            "mean": 0.279153362169902,
+            "std": 0.4026364773930872
+        }
+    },
+    "var-random": {
+        "sufficiency": {
+            "mean": 0.5355305908495315,
+            "std": 0.4197141873409139
+        },
+        "comprehensiveness": {
+            "mean": 0.19971794577601373,
+            "std": 0.3530147506244288
+        }
+    },
+    "var-var-len_var-feat": {
+        "sufficiency": {
+            "mean": 0.6766798219545901,
+            "std": 0.382870334997008
+        },
+        "comprehensiveness": {
+            "mean": 0.34125323093403,
+            "std": 0.4298817545745814
+        }
+    },
+    "var-var-len_var-feat_var-type": {
+        "sufficiency": {
+            "mean": 0.7514209248788171,
+            "std": 0.35845539234219365
+        },
+        "comprehensiveness": {
+            "mean": 0.6846414690430227,
+            "std": 0.4169805156711139
+        }
+    }
+}

double_jsd/faithfulness_metrics/agnews/contigious-test-f1-metrics-description.json

@@ -0,0 +1,42 @@
+{
+    "f1 macro avg - model labels": {
+        "fixed-deeplift": 89.605,
+        "fixed-lime": 92.766,
+        "fixed-attention": 86.536,
+        "fixed-ig": 83.615,
+        "fixed-gradients": 89.662,
+        "fixed-scaled attention": 81.972,
+        "fixed-random": 90.377,
+        "fixed-fixed-len_var-feat": 79.536,
+        "fixed-fixed-len_var-feat_var-type": 40.644,
+        "var-deeplift": 85.655,
+        "var-lime": 89.03,
+        "var-attention": 85.096,
+        "var-ig": 80.928,
+        "var-gradients": 88.672,
+        "var-scaled attention": 79.829,
+        "var-random": 88.024,
+        "var-var-len_var-feat": 76.471,
+        "var-var-len_var-feat_var-type": 40.167
+    },
+    "f1 macro avg - actual labels": {
+        "fixed-deeplift": 86.807,
+        "fixed-lime": 89.374,
+        "fixed-attention": 83.972,
+        "fixed-ig": 82.043,
+        "fixed-gradients": 86.659,
+        "fixed-scaled attention": 81.082,
+        "fixed-random": 87.025,
+        "fixed-fixed-len_var-feat": 78.449,
+        "fixed-fixed-len_var-feat_var-type": 42.421,
+        "var-deeplift": 84.041,
+        "var-lime": 87.098,
+        "var-attention": 83.228,
+        "var-ig": 80.533,
+        "var-gradients": 86.358,
+        "var-scaled attention": 79.964,
+        "var-random": 85.986,
+        "var-var-len_var-feat": 76.764,
+        "var-var-len_var-feat_var-type": 42.642
+    }
+}