From d7008a4e7cc665fcc17beff3b2bcf67faf89327c Mon Sep 17 00:00:00 2001
From: Walter Laurito <lauritowal@yahoo.com>
Date: Fri, 9 Jun 2023 15:45:43 +0200
Subject: [PATCH 01/77] add burns norm

---
 elk/training/burns_norm.py   | 17 +++++++++++++++
 elk/training/ccs_reporter.py | 41 +++++++++++++++++++++---------------
 2 files changed, 41 insertions(+), 17 deletions(-)
 create mode 100644 elk/training/burns_norm.py

diff --git a/elk/training/burns_norm.py b/elk/training/burns_norm.py
new file mode 100644
index 00000000..f25c199a
--- /dev/null
+++ b/elk/training/burns_norm.py
@@ -0,0 +1,17 @@
+import torch
+from torch import Tensor, nn
+
+
+class BurnsNorm(nn.Module):
+    """ Burns et al. style normalization  """
+
+    def forward(self, x: Tensor) -> Tensor:
+        breakpoint()
+        assert x.dim() == 3, "the input should have a dimension of 3." # TODO: add info about needed shape
+
+        print("Per Prompt Normalization...")
+        x: Tensor = x - torch.mean(x, dim=0)
+        norm = torch.linalg.norm(x, dim=2)
+        avg_norm = torch.mean(norm)
+        return x / avg_norm * torch.sqrt(torch.tensor(x.shape[2], dtype=torch.float32))
+
diff --git a/elk/training/ccs_reporter.py b/elk/training/ccs_reporter.py
index f5e5da34..a0aaf5af 100644
--- a/elk/training/ccs_reporter.py
+++ b/elk/training/ccs_reporter.py
@@ -10,6 +10,7 @@
 import torch.nn as nn
 from torch import Tensor
 
+from .burns_norm import BurnsNorm
 from ..metrics import roc_auc
 from ..parsing import parse_loss
 from ..utils.typing import assert_type
@@ -57,6 +58,7 @@ class CcsReporterConfig(ReporterConfig):
     init: Literal["default", "pca", "spherical", "zero"] = "default"
     loss: list[str] = field(default_factory=lambda: ["ccs"])
     loss_dict: dict[str, float] = field(default_factory=dict, init=False)
+    normalization: Literal["lace", "burns"] = "leace"
     num_layers: int = 1
     pre_ln: bool = False
     supervised_weight: float = 0.0
@@ -67,6 +69,7 @@ class CcsReporterConfig(ReporterConfig):
     optimizer: Literal["adam", "lbfgs"] = "lbfgs"
     weight_decay: float = 0.01
 
+
     @classmethod
     def reporter_class(cls) -> type[Reporter]:
         return CcsReporter
@@ -107,13 +110,15 @@ def __init__(
         self.scale = nn.Parameter(torch.ones(1, device=device, dtype=dtype))
 
         hidden_size = cfg.hidden_size or 4 * in_features // 3
-
-        self.norm = ConceptEraser(
-            in_features,
-            2 * num_variants,
-            device=device,
-            dtype=dtype,
-        )
+        if self.config.normalization == "burns":
+            self.norm = BurnsNorm()
+        else:
+            self.norm = ConceptEraser(
+                in_features,
+                2 * num_variants,
+                device=device,
+                dtype=dtype,
+            )
         self.probe = nn.Sequential(
             nn.Linear(
                 in_features,
@@ -262,16 +267,18 @@ def fit(self, hiddens: Tensor) -> float:
         n, v, _ = x_neg.shape
         prompt_ids = torch.eye(v, device=x_neg.device).expand(n, -1, -1)
 
-        self.norm.update(
-            x=x_neg,
-            # Independent indicator for each (template, pseudo-label) pair
-            y=torch.cat([torch.zeros_like(prompt_ids), prompt_ids], dim=-1),
-        )
-        self.norm.update(
-            x=x_pos,
-            # Independent indicator for each (template, pseudo-label) pair
-            y=torch.cat([prompt_ids, torch.zeros_like(prompt_ids)], dim=-1),
-        )
+        if self.config.normalization == "leace":
+            self.norm.update(
+                x=x_neg,
+                # Independent indicator for each (template, pseudo-label) pair
+                y=torch.cat([torch.zeros_like(prompt_ids), prompt_ids], dim=-1),
+            )
+            self.norm.update(
+                x=x_pos,
+                # Independent indicator for each (template, pseudo-label) pair
+                y=torch.cat([prompt_ids, torch.zeros_like(prompt_ids)], dim=-1),
+            )
+
         x_neg, x_pos = self.norm(x_neg), self.norm(x_pos)
 
         # Record the best acc, loss, and params found so far

From b22d15562601f0c10600216d5717ff71cba75328 Mon Sep 17 00:00:00 2001
From: Walter Laurito <lauritowal@yahoo.com>
Date: Thu, 15 Jun 2023 13:12:55 +0200
Subject: [PATCH 02/77] cleanup class and add annotation

---
 elk/training/burns_norm.py | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/elk/training/burns_norm.py b/elk/training/burns_norm.py
index f25c199a..d0266c40 100644
--- a/elk/training/burns_norm.py
+++ b/elk/training/burns_norm.py
@@ -3,15 +3,12 @@
 
 
 class BurnsNorm(nn.Module):
-    """ Burns et al. style normalization  """
+    """ Burns et al. style normalization  Minimal changes from the original code. """
 
-    def forward(self, x: Tensor) -> Tensor:
-        breakpoint()
-        assert x.dim() == 3, "the input should have a dimension of 3." # TODO: add info about needed shape
+    def forward(self, x: Tensor) -> Tensor:        
+        assert x.dim() == 3, f"the input should have a dimension of 3 not dimension {x.dim()}, shape of x: {x.shape}" 
 
-        print("Per Prompt Normalization...")
         x: Tensor = x - torch.mean(x, dim=0)
         norm = torch.linalg.norm(x, dim=2)
         avg_norm = torch.mean(norm)
-        return x / avg_norm * torch.sqrt(torch.tensor(x.shape[2], dtype=torch.float32))
-
+        return x / avg_norm * torch.sqrt(torch.tensor(x.shape[2], dtype=torch.float32))
\ No newline at end of file

From 29bd0af660c32d752705255d8665bf68a2502555 Mon Sep 17 00:00:00 2001
From: Walter Laurito <lauritowal@yahoo.com>
Date: Thu, 15 Jun 2023 13:13:18 +0200
Subject: [PATCH 03/77] shorten arg name to norm

---
 elk/training/ccs_reporter.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/elk/training/ccs_reporter.py b/elk/training/ccs_reporter.py
index a0aaf5af..b04c55cf 100644
--- a/elk/training/ccs_reporter.py
+++ b/elk/training/ccs_reporter.py
@@ -58,7 +58,7 @@ class CcsReporterConfig(ReporterConfig):
     init: Literal["default", "pca", "spherical", "zero"] = "default"
     loss: list[str] = field(default_factory=lambda: ["ccs"])
     loss_dict: dict[str, float] = field(default_factory=dict, init=False)
-    normalization: Literal["lace", "burns"] = "leace"
+    norm: Literal["leace", "burns"] = "leace" # TODO: move to parent class ?
     num_layers: int = 1
     pre_ln: bool = False
     supervised_weight: float = 0.0
@@ -110,8 +110,9 @@ def __init__(
         self.scale = nn.Parameter(torch.ones(1, device=device, dtype=dtype))
 
         hidden_size = cfg.hidden_size or 4 * in_features // 3
-        if self.config.normalization == "burns":
+        if self.config.norm == "burns":
             self.norm = BurnsNorm()
+            print("buuurn")
         else:
             self.norm = ConceptEraser(
                 in_features,
@@ -236,7 +237,7 @@ def reset_parameters(self):
 
     def forward(self, x: Tensor) -> Tensor:
         """Return the credence assigned to the hidden state `x`."""
-        raw_scores = self.probe(self.norm(x)).squeeze(-1)
+        raw_scores = self.probe(x).squeeze(-1)
         return raw_scores.mul(self.scale).add(self.bias).squeeze(-1)
 
     def loss(self, logit0: Tensor, logit1: Tensor) -> Tensor:
@@ -267,7 +268,7 @@ def fit(self, hiddens: Tensor) -> float:
         n, v, _ = x_neg.shape
         prompt_ids = torch.eye(v, device=x_neg.device).expand(n, -1, -1)
 
-        if self.config.normalization == "leace":
+        if self.config.norm == "leace":
             self.norm.update(
                 x=x_neg,
                 # Independent indicator for each (template, pseudo-label) pair
@@ -278,7 +279,7 @@ def fit(self, hiddens: Tensor) -> float:
                 # Independent indicator for each (template, pseudo-label) pair
                 y=torch.cat([prompt_ids, torch.zeros_like(prompt_ids)], dim=-1),
             )
-
+            
         x_neg, x_pos = self.norm(x_neg), self.norm(x_pos)
 
         # Record the best acc, loss, and params found so far

From 87ed4b85260d710a0a8e40b6bd0497dde722f015 Mon Sep 17 00:00:00 2001
From: Walter Laurito <lauritowal@yahoo.com>
Date: Thu, 15 Jun 2023 18:05:19 +0200
Subject: [PATCH 04/77] cleanup comment

---
 elk/training/burns_norm.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/elk/training/burns_norm.py b/elk/training/burns_norm.py
index d0266c40..34de0c83 100644
--- a/elk/training/burns_norm.py
+++ b/elk/training/burns_norm.py
@@ -6,7 +6,7 @@ class BurnsNorm(nn.Module):
     """ Burns et al. style normalization  Minimal changes from the original code. """
 
     def forward(self, x: Tensor) -> Tensor:        
-        assert x.dim() == 3, f"the input should have a dimension of 3 not dimension {x.dim()}, shape of x: {x.shape}" 
+        assert x.dim() == 3, f"the input should have a dimension of 3 not dimension {x.dim()}, current shape of input x: {x.shape}" 
 
         x: Tensor = x - torch.mean(x, dim=0)
         norm = torch.linalg.norm(x, dim=2)

From bd7dfcebb22de2581dbab82d359a39a9298c2f67 Mon Sep 17 00:00:00 2001
From: Walter Laurito <lauritowal@yahoo.com>
Date: Thu, 15 Jun 2023 18:07:53 +0200
Subject: [PATCH 05/77] remove print

---
 elk/training/ccs_reporter.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/elk/training/ccs_reporter.py b/elk/training/ccs_reporter.py
index b04c55cf..e83d2f37 100644
--- a/elk/training/ccs_reporter.py
+++ b/elk/training/ccs_reporter.py
@@ -112,7 +112,6 @@ def __init__(
         hidden_size = cfg.hidden_size or 4 * in_features // 3
         if self.config.norm == "burns":
             self.norm = BurnsNorm()
-            print("buuurn")
         else:
             self.norm = ConceptEraser(
                 in_features,

From 28d6044f897a10751fbce8c8ac0e47bdb5defce5 Mon Sep 17 00:00:00 2001
From: Walter Laurito <lauritowal@yahoo.com>
Date: Thu, 15 Jun 2023 18:10:15 +0200
Subject: [PATCH 06/77] remove comment

---
 elk/training/ccs_reporter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/elk/training/ccs_reporter.py b/elk/training/ccs_reporter.py
index e83d2f37..a410425f 100644
--- a/elk/training/ccs_reporter.py
+++ b/elk/training/ccs_reporter.py
@@ -58,7 +58,7 @@ class CcsReporterConfig(ReporterConfig):
     init: Literal["default", "pca", "spherical", "zero"] = "default"
     loss: list[str] = field(default_factory=lambda: ["ccs"])
     loss_dict: dict[str, float] = field(default_factory=dict, init=False)
-    norm: Literal["leace", "burns"] = "leace" # TODO: move to parent class ?
+    norm: Literal["leace", "burns"] = "leace"
     num_layers: int = 1
     pre_ln: bool = False
     supervised_weight: float = 0.0

From cd8fdc00cafb151f444c5ec7df3e58604076c97d Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Thu, 15 Jun 2023 16:34:58 +0000
Subject: [PATCH 07/77] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 elk/training/burns_norm.py   | 10 ++++++----
 elk/training/ccs_reporter.py |  5 ++---
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/elk/training/burns_norm.py b/elk/training/burns_norm.py
index 34de0c83..39bc8f64 100644
--- a/elk/training/burns_norm.py
+++ b/elk/training/burns_norm.py
@@ -3,12 +3,14 @@
 
 
 class BurnsNorm(nn.Module):
-    """ Burns et al. style normalization  Minimal changes from the original code. """
+    """Burns et al. style normalization  Minimal changes from the original code."""
 
-    def forward(self, x: Tensor) -> Tensor:        
-        assert x.dim() == 3, f"the input should have a dimension of 3 not dimension {x.dim()}, current shape of input x: {x.shape}" 
+    def forward(self, x: Tensor) -> Tensor:
+        assert (
+            x.dim() == 3
+        ), f"the input should have a dimension of 3 not dimension {x.dim()}, current shape of input x: {x.shape}"
 
         x: Tensor = x - torch.mean(x, dim=0)
         norm = torch.linalg.norm(x, dim=2)
         avg_norm = torch.mean(norm)
-        return x / avg_norm * torch.sqrt(torch.tensor(x.shape[2], dtype=torch.float32))
\ No newline at end of file
+        return x / avg_norm * torch.sqrt(torch.tensor(x.shape[2], dtype=torch.float32))
diff --git a/elk/training/ccs_reporter.py b/elk/training/ccs_reporter.py
index a410425f..ff66b3c9 100644
--- a/elk/training/ccs_reporter.py
+++ b/elk/training/ccs_reporter.py
@@ -10,10 +10,10 @@
 import torch.nn as nn
 from torch import Tensor
 
-from .burns_norm import BurnsNorm
 from ..metrics import roc_auc
 from ..parsing import parse_loss
 from ..utils.typing import assert_type
+from .burns_norm import BurnsNorm
 from .classifier import Classifier
 from .concept_eraser import ConceptEraser
 from .losses import LOSSES
@@ -69,7 +69,6 @@ class CcsReporterConfig(ReporterConfig):
     optimizer: Literal["adam", "lbfgs"] = "lbfgs"
     weight_decay: float = 0.01
 
-
     @classmethod
     def reporter_class(cls) -> type[Reporter]:
         return CcsReporter
@@ -278,7 +277,7 @@ def fit(self, hiddens: Tensor) -> float:
                 # Independent indicator for each (template, pseudo-label) pair
                 y=torch.cat([prompt_ids, torch.zeros_like(prompt_ids)], dim=-1),
             )
-            
+
         x_neg, x_pos = self.norm(x_neg), self.norm(x_pos)
 
         # Record the best acc, loss, and params found so far

From cc2dbd547ad227f83bf8a1e001a941dbaa046f01 Mon Sep 17 00:00:00 2001
From: Walter Laurito <lauritowal@yahoo.com>
Date: Thu, 15 Jun 2023 18:53:12 +0200
Subject: [PATCH 08/77] pre-commit cleanup

---
 .pre-commit-config.yaml      |  2 +-
 elk/training/burns_norm.py   | 11 +++++++----
 elk/training/ccs_reporter.py |  7 +++----
 3 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 04929ca3..ded2fbfe 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -24,4 +24,4 @@ repos:
     hooks:
     -   id: codespell
         # The promptsource templates spuriously get flagged without this
-        args: ["-L fpr", "--skip=*.yaml"]
+        args: ["-L fpr,leace", "--skip=*.yaml"]
diff --git a/elk/training/burns_norm.py b/elk/training/burns_norm.py
index 34de0c83..2e76b6e6 100644
--- a/elk/training/burns_norm.py
+++ b/elk/training/burns_norm.py
@@ -3,12 +3,15 @@
 
 
 class BurnsNorm(nn.Module):
-    """ Burns et al. style normalization  Minimal changes from the original code. """
+    """Burns et al. style normalization  Minimal changes from the original code."""
 
-    def forward(self, x: Tensor) -> Tensor:        
-        assert x.dim() == 3, f"the input should have a dimension of 3 not dimension {x.dim()}, current shape of input x: {x.shape}" 
+    def forward(self, x: Tensor) -> Tensor:
+        assert (
+            x.dim() == 3
+        ), f"the input should have a dimension of 3 not dimension {x.dim()}, \
+        current shape of input x: {x.shape}"
 
         x: Tensor = x - torch.mean(x, dim=0)
         norm = torch.linalg.norm(x, dim=2)
         avg_norm = torch.mean(norm)
-        return x / avg_norm * torch.sqrt(torch.tensor(x.shape[2], dtype=torch.float32))
\ No newline at end of file
+        return x / avg_norm * torch.sqrt(torch.tensor(x.shape[2], dtype=torch.float32))
diff --git a/elk/training/ccs_reporter.py b/elk/training/ccs_reporter.py
index a410425f..f434894c 100644
--- a/elk/training/ccs_reporter.py
+++ b/elk/training/ccs_reporter.py
@@ -10,10 +10,10 @@
 import torch.nn as nn
 from torch import Tensor
 
-from .burns_norm import BurnsNorm
 from ..metrics import roc_auc
 from ..parsing import parse_loss
 from ..utils.typing import assert_type
+from .burns_norm import BurnsNorm
 from .classifier import Classifier
 from .concept_eraser import ConceptEraser
 from .losses import LOSSES
@@ -58,7 +58,7 @@ class CcsReporterConfig(ReporterConfig):
     init: Literal["default", "pca", "spherical", "zero"] = "default"
     loss: list[str] = field(default_factory=lambda: ["ccs"])
     loss_dict: dict[str, float] = field(default_factory=dict, init=False)
-    norm: Literal["leace", "burns"] = "leace"
+    norm: Literal["leace", "burns"] = "leace"  # codespell: ignore
     num_layers: int = 1
     pre_ln: bool = False
     supervised_weight: float = 0.0
@@ -69,7 +69,6 @@ class CcsReporterConfig(ReporterConfig):
     optimizer: Literal["adam", "lbfgs"] = "lbfgs"
     weight_decay: float = 0.01
 
-
     @classmethod
     def reporter_class(cls) -> type[Reporter]:
         return CcsReporter
@@ -278,7 +277,7 @@ def fit(self, hiddens: Tensor) -> float:
                 # Independent indicator for each (template, pseudo-label) pair
                 y=torch.cat([prompt_ids, torch.zeros_like(prompt_ids)], dim=-1),
             )
-            
+
         x_neg, x_pos = self.norm(x_neg), self.norm(x_pos)
 
         # Record the best acc, loss, and params found so far

From 8e65e2dd0710e6e12ac6a77184ef08bc8fa75e70 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Fri, 16 Jun 2023 11:11:28 +0000
Subject: [PATCH 09/77] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 elk/extraction/extraction.py | 4 ++--
 elk/training/burns_norm.py   | 8 ++++++--
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/elk/extraction/extraction.py b/elk/extraction/extraction.py
index 615faebf..fbb93763 100644
--- a/elk/extraction/extraction.py
+++ b/elk/extraction/extraction.py
@@ -295,7 +295,7 @@ def extract_hiddens(
                 )
                 # Throw out layers we don't care about
                 hiddens = [hiddens[i] for i in layer_indices]
-                
+
                 # Current shape of each element: (batch_size, seq_len, hidden_size)
                 if cfg.token_loc == "first":
                     hiddens = [h[..., 0, :] for h in hiddens]
@@ -320,7 +320,7 @@ def extract_hiddens(
         # We skipped a variant because it was too long; move on to the next example
         if len(text_questions) != num_variants:
             continue
-        
+
         out_record: dict[str, Any] = dict(
             label=example["label"],
             variant_ids=example["template_names"],
diff --git a/elk/training/burns_norm.py b/elk/training/burns_norm.py
index 62b7e477..d9b1281e 100644
--- a/elk/training/burns_norm.py
+++ b/elk/training/burns_norm.py
@@ -10,7 +10,7 @@ def forward(self, x: Tensor) -> Tensor:
             x.dim() == 3
         ), f"the input should have a dimension of 3 not dimension {x.dim()}, \
         current shape of input x: {x.shape}"
-        
+
         x_mean: Tensor = x - torch.mean(x, dim=0)
         if torch.all(x_mean == 0):
             # input embeddings entries are identical, which leads to x_mean having only zero entries.
@@ -18,4 +18,8 @@ def forward(self, x: Tensor) -> Tensor:
         else:
             norm = torch.linalg.norm(x_mean, dim=2)
             avg_norm = torch.mean(norm)
-            return x_mean / avg_norm * torch.sqrt(torch.tensor(x_mean.shape[2], dtype=torch.float32))
+            return (
+                x_mean
+                / avg_norm
+                * torch.sqrt(torch.tensor(x_mean.shape[2], dtype=torch.float32))
+            )

From a3b8c3cb8ce08d3d2065985264b2a0b29cd3250a Mon Sep 17 00:00:00 2001
From: Walter Laurito <Lauritowal@yahoo.com>
Date: Fri, 16 Jun 2023 14:06:50 +0100
Subject: [PATCH 10/77] Update ccs_reporter.py

remove ignore
---
 elk/training/ccs_reporter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/elk/training/ccs_reporter.py b/elk/training/ccs_reporter.py
index f434894c..ff66b3c9 100644
--- a/elk/training/ccs_reporter.py
+++ b/elk/training/ccs_reporter.py
@@ -58,7 +58,7 @@ class CcsReporterConfig(ReporterConfig):
     init: Literal["default", "pca", "spherical", "zero"] = "default"
     loss: list[str] = field(default_factory=lambda: ["ccs"])
     loss_dict: dict[str, float] = field(default_factory=dict, init=False)
-    norm: Literal["leace", "burns"] = "leace"  # codespell: ignore
+    norm: Literal["leace", "burns"] = "leace"
     num_layers: int = 1
     pre_ln: bool = False
     supervised_weight: float = 0.0

From 73bfa32b0b34ad7eeca8d68393db95749cd0a997 Mon Sep 17 00:00:00 2001
From: Walter Laurito <lauritowal@yahoo.com>
Date: Fri, 16 Jun 2023 16:25:29 +0200
Subject: [PATCH 11/77] add nn.module

---
 elk/training/ccs_reporter.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/elk/training/ccs_reporter.py b/elk/training/ccs_reporter.py
index ff66b3c9..57f3addd 100644
--- a/elk/training/ccs_reporter.py
+++ b/elk/training/ccs_reporter.py
@@ -110,9 +110,9 @@ def __init__(
 
         hidden_size = cfg.hidden_size or 4 * in_features // 3
         if self.config.norm == "burns":
-            self.norm = BurnsNorm()
+            self.norm: nn.module = BurnsNorm()
         else:
-            self.norm = ConceptEraser(
+            self.norm: nn.module = ConceptEraser(
                 in_features,
                 2 * num_variants,
                 device=device,

From fa8751bcf0ca7e403107b8239567e9864a956f5d Mon Sep 17 00:00:00 2001
From: Walter Laurito <Lauritowal@yahoo.com>
Date: Thu, 22 Jun 2023 11:02:04 +0000
Subject: [PATCH 12/77] correct annotation for norm

---
 elk/training/ccs_reporter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/elk/training/ccs_reporter.py b/elk/training/ccs_reporter.py
index 57f3addd..225dda05 100644
--- a/elk/training/ccs_reporter.py
+++ b/elk/training/ccs_reporter.py
@@ -38,7 +38,7 @@ class CcsReporterConfig(ReporterConfig):
             Example: --loss 1.0*consistency_squared 0.5*prompt_var
             corresponds to the loss function 1.0*consistency_squared + 0.5*prompt_var.
             Defaults to the loss "ccs_squared_loss".
-        normalization: The kind of normalization to apply to the hidden states.
+        norm: The kind of normalization to apply to the hidden states.
         num_layers: The number of layers in the MLP. Defaults to 1.
         pre_ln: Whether to include a LayerNorm module before the first linear
             layer. Defaults to False.

From a04f85215f847b005e50346ab77e530cab9731ac Mon Sep 17 00:00:00 2001
From: jon <jonathan.n@u.nus.edu>
Date: Mon, 26 Jun 2023 14:58:49 +0100
Subject: [PATCH 13/77] remove erroring type annotation

---
 elk/training/burns_norm.py   | 3 ++-
 elk/training/ccs_reporter.py | 5 +++--
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/elk/training/burns_norm.py b/elk/training/burns_norm.py
index d9b1281e..dec6d90b 100644
--- a/elk/training/burns_norm.py
+++ b/elk/training/burns_norm.py
@@ -13,7 +13,8 @@ def forward(self, x: Tensor) -> Tensor:
 
         x_mean: Tensor = x - torch.mean(x, dim=0)
         if torch.all(x_mean == 0):
-            # input embeddings entries are identical, which leads to x_mean having only zero entries.
+            # input embeddings entries are identical,
+            # which leads to x_mean having only zero entries.
             return x
         else:
             norm = torch.linalg.norm(x_mean, dim=2)
diff --git a/elk/training/ccs_reporter.py b/elk/training/ccs_reporter.py
index 225dda05..a40bd7b9 100644
--- a/elk/training/ccs_reporter.py
+++ b/elk/training/ccs_reporter.py
@@ -109,10 +109,11 @@ def __init__(
         self.scale = nn.Parameter(torch.ones(1, device=device, dtype=dtype))
 
         hidden_size = cfg.hidden_size or 4 * in_features // 3
+
         if self.config.norm == "burns":
-            self.norm: nn.module = BurnsNorm()
+            self.norm = BurnsNorm()
         else:
-            self.norm: nn.module = ConceptEraser(
+            self.norm = ConceptEraser(
                 in_features,
                 2 * num_variants,
                 device=device,

From dffbe6808c31f18bb29ed564ae0e7260cce77e70 Mon Sep 17 00:00:00 2001
From: jon <jonathan.n@u.nus.edu>
Date: Mon, 26 Jun 2023 16:44:25 +0100
Subject: [PATCH 14/77] patch over bug with pyright

---
 elk/training/ccs_reporter.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/elk/training/ccs_reporter.py b/elk/training/ccs_reporter.py
index a40bd7b9..5d3008d2 100644
--- a/elk/training/ccs_reporter.py
+++ b/elk/training/ccs_reporter.py
@@ -268,12 +268,13 @@ def fit(self, hiddens: Tensor) -> float:
         prompt_ids = torch.eye(v, device=x_neg.device).expand(n, -1, -1)
 
         if self.config.norm == "leace":
-            self.norm.update(
+            # type ignore because otherwise throws error, probably bug with pyright
+            self.norm.update(  # type: ignore
                 x=x_neg,
                 # Independent indicator for each (template, pseudo-label) pair
                 y=torch.cat([torch.zeros_like(prompt_ids), prompt_ids], dim=-1),
             )
-            self.norm.update(
+            self.norm.update(  # type: ignore
                 x=x_pos,
                 # Independent indicator for each (template, pseudo-label) pair
                 y=torch.cat([prompt_ids, torch.zeros_like(prompt_ids)], dim=-1),

From 80826c8f5a27d24dff4ddcf19ec0d76bb814bb7a Mon Sep 17 00:00:00 2001
From: jon <jonathan.n@u.nus.edu>
Date: Fri, 30 Jun 2023 11:47:59 +0100
Subject: [PATCH 15/77] remove ignores

---
 elk/training/ccs_reporter.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/elk/training/ccs_reporter.py b/elk/training/ccs_reporter.py
index 5d3008d2..6397a24b 100644
--- a/elk/training/ccs_reporter.py
+++ b/elk/training/ccs_reporter.py
@@ -266,15 +266,13 @@ def fit(self, hiddens: Tensor) -> float:
         # One-hot indicators for each prompt template
         n, v, _ = x_neg.shape
         prompt_ids = torch.eye(v, device=x_neg.device).expand(n, -1, -1)
-
-        if self.config.norm == "leace":
-            # type ignore because otherwise throws error, probably bug with pyright
-            self.norm.update(  # type: ignore
+        if isinstance(self.norm, ConceptEraser):
+            self.norm.update(
                 x=x_neg,
                 # Independent indicator for each (template, pseudo-label) pair
                 y=torch.cat([torch.zeros_like(prompt_ids), prompt_ids], dim=-1),
             )
-            self.norm.update(  # type: ignore
+            self.norm.update(
                 x=x_pos,
                 # Independent indicator for each (template, pseudo-label) pair
                 y=torch.cat([prompt_ids, torch.zeros_like(prompt_ids)], dim=-1),

From 70c737e750377e9ef98f40f0fc1ae9041b16e21b Mon Sep 17 00:00:00 2001
From: jon <jonathan.n@u.nus.edu>
Date: Mon, 3 Jul 2023 18:42:12 +0000
Subject: [PATCH 16/77] fix nans better

---
 elk/training/burns_norm.py | 20 ++++++++------------
 1 file changed, 8 insertions(+), 12 deletions(-)

diff --git a/elk/training/burns_norm.py b/elk/training/burns_norm.py
index dec6d90b..b98a5f35 100644
--- a/elk/training/burns_norm.py
+++ b/elk/training/burns_norm.py
@@ -12,15 +12,11 @@ def forward(self, x: Tensor) -> Tensor:
         current shape of input x: {x.shape}"
 
         x_mean: Tensor = x - torch.mean(x, dim=0)
-        if torch.all(x_mean == 0):
-            # input embeddings entries are identical,
-            # which leads to x_mean having only zero entries.
-            return x
-        else:
-            norm = torch.linalg.norm(x_mean, dim=2)
-            avg_norm = torch.mean(norm)
-            return (
-                x_mean
-                / avg_norm
-                * torch.sqrt(torch.tensor(x_mean.shape[2], dtype=torch.float32))
-            )
+        norm = torch.linalg.norm(x_mean, dim=2)
+        avg_norm = torch.mean(norm)
+        eps = torch.finfo(x.dtype).eps
+        return (
+            x_mean
+            / (avg_norm + eps)  # to avoid division by zero
+            * torch.sqrt(torch.tensor(x_mean.shape[2], dtype=torch.float32))
+        )

From 92f9086e4f0161414fec2adb9c5235fd354e7ccc Mon Sep 17 00:00:00 2001
From: Walter Laurito <Lauritowal@yahoo.com>
Date: Wed, 19 Jul 2023 17:35:41 +0000
Subject: [PATCH 17/77] remove checking for embeddings

---
 elk/training/burns_norm.py | 18 +++++++-----------
 1 file changed, 7 insertions(+), 11 deletions(-)

diff --git a/elk/training/burns_norm.py b/elk/training/burns_norm.py
index d9b1281e..4fea3231 100644
--- a/elk/training/burns_norm.py
+++ b/elk/training/burns_norm.py
@@ -12,14 +12,10 @@ def forward(self, x: Tensor) -> Tensor:
         current shape of input x: {x.shape}"
 
         x_mean: Tensor = x - torch.mean(x, dim=0)
-        if torch.all(x_mean == 0):
-            # input embeddings entries are identical, which leads to x_mean having only zero entries.
-            return x
-        else:
-            norm = torch.linalg.norm(x_mean, dim=2)
-            avg_norm = torch.mean(norm)
-            return (
-                x_mean
-                / avg_norm
-                * torch.sqrt(torch.tensor(x_mean.shape[2], dtype=torch.float32))
-            )
+        norm = torch.linalg.norm(x_mean, dim=2)
+        avg_norm = torch.mean(norm)
+        return (
+            x_mean
+            / avg_norm
+            * torch.sqrt(torch.tensor(x_mean.shape[2], dtype=torch.float32))
+        )

From 9027a925f26d571fd7ce3f3a4bb03baaba85b1a8 Mon Sep 17 00:00:00 2001
From: Walter Laurito <Lauritowal@yahoo.com>
Date: Wed, 19 Jul 2023 18:53:48 +0000
Subject: [PATCH 18/77] fix precommit stuff

---
 elk/training/ccs_reporter.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/elk/training/ccs_reporter.py b/elk/training/ccs_reporter.py
index 09860314..7e80063d 100644
--- a/elk/training/ccs_reporter.py
+++ b/elk/training/ccs_reporter.py
@@ -13,7 +13,6 @@
 from ..parsing import parse_loss
 from ..utils.typing import assert_type
 from .burns_norm import BurnsNorm
-from .classifier import Classifier
 from .common import FitterConfig
 from .losses import LOSSES
 from .platt_scaling import PlattMixin

From 4ae416a7297adc37fee6273257b4a0e008dfac3c Mon Sep 17 00:00:00 2001
From: Walter Laurito <Lauritowal@yahoo.com>
Date: Wed, 19 Jul 2023 19:55:46 +0000
Subject: [PATCH 19/77] readd self.norm

---
 elk/training/ccs_reporter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/elk/training/ccs_reporter.py b/elk/training/ccs_reporter.py
index 7e80063d..ad56e858 100644
--- a/elk/training/ccs_reporter.py
+++ b/elk/training/ccs_reporter.py
@@ -164,7 +164,7 @@ def reset_parameters(self):
     def forward(self, x: Tensor) -> Tensor:
         """Return the credence assigned to the hidden state `x`."""
         assert self.norm is not None, "Must call fit() before forward()"
-        raw_scores = self.probe(x).squeeze(-1)
+        raw_scores = self.probe(self.norm(x)).squeeze(-1)
         return raw_scores.mul(self.scale).add(self.bias).squeeze(-1)
 
     def loss(self, logit0: Tensor, logit1: Tensor) -> Tensor:

From 55c867ec448212a140d1f4f6ffbc52223d546493 Mon Sep 17 00:00:00 2001
From: Walter Laurito <Lauritowal@yahoo.com>
Date: Wed, 19 Jul 2023 19:56:37 +0000
Subject: [PATCH 20/77] readd space

---
 elk/training/ccs_reporter.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/elk/training/ccs_reporter.py b/elk/training/ccs_reporter.py
index ad56e858..39d4502e 100644
--- a/elk/training/ccs_reporter.py
+++ b/elk/training/ccs_reporter.py
@@ -164,6 +164,7 @@ def reset_parameters(self):
     def forward(self, x: Tensor) -> Tensor:
         """Return the credence assigned to the hidden state `x`."""
         assert self.norm is not None, "Must call fit() before forward()"
+        
         raw_scores = self.probe(self.norm(x)).squeeze(-1)
         return raw_scores.mul(self.scale).add(self.bias).squeeze(-1)
 

From b257c97b219e2494f7a9785e873f3f852b2a7f1e Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Wed, 19 Jul 2023 19:56:52 +0000
Subject: [PATCH 21/77] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 elk/training/ccs_reporter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/elk/training/ccs_reporter.py b/elk/training/ccs_reporter.py
index 39d4502e..01b9670c 100644
--- a/elk/training/ccs_reporter.py
+++ b/elk/training/ccs_reporter.py
@@ -164,7 +164,7 @@ def reset_parameters(self):
     def forward(self, x: Tensor) -> Tensor:
         """Return the credence assigned to the hidden state `x`."""
         assert self.norm is not None, "Must call fit() before forward()"
-        
+
         raw_scores = self.probe(self.norm(x)).squeeze(-1)
         return raw_scores.mul(self.scale).add(self.bias).squeeze(-1)
 

From 4fc5c6a7462eb95a06da2932672f2ef09bfa8cd6 Mon Sep 17 00:00:00 2001
From: Reagan Lee <reaganjlee@gmail.com>
Date: Tue, 25 Jul 2023 14:50:33 +0000
Subject: [PATCH 22/77] fix sweep ds config bug

---
 elk/training/sweep.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/elk/training/sweep.py b/elk/training/sweep.py
index e4aca5a0..a4e5c97a 100755
--- a/elk/training/sweep.py
+++ b/elk/training/sweep.py
@@ -22,7 +22,8 @@ def assert_models_exist(model_names):
 
 def assert_datasets_exist(dataset_names):
     for dataset_name in dataset_names:
-        get_dataset_config_info(dataset_name)
+        ds_name, _, config_name = dataset_name.partition(":")
+        get_dataset_config_info(ds_name, config_name=config_name)
 
 
 @dataclass

From 5bd92e8a736b4e974a00b8b3e2dd5dd6638e2c3e Mon Sep 17 00:00:00 2001
From: Reagan Lee <reaganjlee@gmail.com>
Date: Tue, 25 Jul 2023 15:53:02 +0000
Subject: [PATCH 23/77] fix transfer_df read bug + make func clearer

---
 elk/plotting/visualize.py | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/elk/plotting/visualize.py b/elk/plotting/visualize.py
index a82fd7d7..bcb99933 100644
--- a/elk/plotting/visualize.py
+++ b/elk/plotting/visualize.py
@@ -216,26 +216,26 @@ def collect(cls, model_path: Path) -> "ModelVisualization":
         model_name = model_path.name
         is_transfer = False
 
-        def get_train_dirs(model_path):
+        def get_eval_dirs(model_path):
             # toplevel is either repo/dataset or dataset
             for toplevel in model_path.iterdir():
                 if (toplevel / "eval.csv").exists():
                     yield toplevel
                 else:
-                    for train_dir in toplevel.iterdir():
-                        yield train_dir
+                    for eval_dir in toplevel.iterdir():
+                        yield eval_dir
 
-        for train_dir in get_train_dirs(model_path):
+        for train_dir in get_eval_dirs(model_path):
             eval_df = cls._read_eval_csv(train_dir, train_dir.name, train_dir.name)
             df = pd.concat([df, eval_df], ignore_index=True)
             transfer_dir = train_dir / "transfer"
             if transfer_dir.exists():
                 is_transfer = True
-                for eval_ds_dir in transfer_dir.iterdir():
-                    eval_df = cls._read_eval_csv(
-                        eval_ds_dir, eval_ds_dir.name, train_dir.name
+                for tfr_ds_dir in get_eval_dirs(transfer_dir):
+                    tfr_df = cls._read_eval_csv(
+                        tfr_ds_dir, tfr_ds_dir.name, train_dir.name
                     )
-                    df = pd.concat([df, eval_df], ignore_index=True)
+                    df = pd.concat([df, tfr_df], ignore_index=True)
 
         df["model_name"] = model_name
         return cls(df, model_name, is_transfer)
@@ -273,6 +273,8 @@ def render_and_save(
     @staticmethod
     def _read_eval_csv(path, eval_dataset, train_dataset):
         file = path / "eval.csv"
+        # if not os.path.exists(file):
+        # file = path / train_dataset / "eval.csv"
         eval_df = pd.read_csv(file)
         eval_df["eval_dataset"] = eval_dataset
         eval_df["train_dataset"] = train_dataset

From c459e8cb30c6a68f321fa301dfe555107bfa22c2 Mon Sep 17 00:00:00 2001
From: Reagan Lee <reaganjlee@gmail.com>
Date: Tue, 25 Jul 2023 16:06:17 +0000
Subject: [PATCH 24/77] cleanup

---
 elk/plotting/visualize.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/elk/plotting/visualize.py b/elk/plotting/visualize.py
index bcb99933..fa183e5a 100644
--- a/elk/plotting/visualize.py
+++ b/elk/plotting/visualize.py
@@ -273,8 +273,6 @@ def render_and_save(
     @staticmethod
     def _read_eval_csv(path, eval_dataset, train_dataset):
         file = path / "eval.csv"
-        # if not os.path.exists(file):
-        # file = path / train_dataset / "eval.csv"
         eval_df = pd.read_csv(file)
         eval_df["eval_dataset"] = eval_dataset
         eval_df["train_dataset"] = train_dataset

From be08c5b6a28ce5e4be4afaf49e43a8232fdcb586 Mon Sep 17 00:00:00 2001
From: Alex Mallen <35092692+AlexTMallen@users.noreply.github.com>
Date: Thu, 27 Jul 2023 09:52:25 -0700
Subject: [PATCH 25/77] Update README.md

Add pypi reference
---
 README.md | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/README.md b/README.md
index e16eb45b..fa74d900 100644
--- a/README.md
+++ b/README.md
@@ -21,8 +21,7 @@ Our code is based on [PyTorch](http://pytorch.org)
 and [Huggingface Transformers](https://huggingface.co/docs/transformers/index). We test the code on Python 3.10 and
 3.11.
 
-First install the package with `pip install -e .` in the root directory, or `pip install -e .[dev]` if you'd like to
-contribute to the project (see **Development** section below). This should install all the necessary dependencies.
+First install the package with `pip install -e .` in the root directory, or `pip install eleuther-elk` to install from PyPi. Use `pip install -e .[dev]` if you'd like to contribute to the project (see **Development** section below). This should install all the necessary dependencies.
 
 To fit reporters for the HuggingFace model `model` and dataset `dataset`, just run:
 

From 6a12fdd00cead712540e28c387357aa790eb4112 Mon Sep 17 00:00:00 2001
From: Walter Laurito <Lauritowal@yahoo.com>
Date: Fri, 28 Jul 2023 15:59:07 +0000
Subject: [PATCH 26/77] fix burns norm to really normalize by prompt

---
 elk/training/burns_norm.py | 47 ++++++++++++++++++++++++++------------
 1 file changed, 32 insertions(+), 15 deletions(-)

diff --git a/elk/training/burns_norm.py b/elk/training/burns_norm.py
index 4fea3231..c337841f 100644
--- a/elk/training/burns_norm.py
+++ b/elk/training/burns_norm.py
@@ -1,21 +1,38 @@
+from dataclasses import dataclass
 import torch
 from torch import Tensor, nn
 
-
 class BurnsNorm(nn.Module):
-    """Burns et al. style normalization  Minimal changes from the original code."""
+    """Burns et al. style normalization. Minimal changes from the original code."""
+    
+    def __init__(self, scale:bool=True):
+        super().__init__()
+        self.scale: bool = scale
 
     def forward(self, x: Tensor) -> Tensor:
-        assert (
-            x.dim() == 3
-        ), f"the input should have a dimension of 3 not dimension {x.dim()}, \
-        current shape of input x: {x.shape}"
-
-        x_mean: Tensor = x - torch.mean(x, dim=0)
-        norm = torch.linalg.norm(x_mean, dim=2)
-        avg_norm = torch.mean(norm)
-        return (
-            x_mean
-            / avg_norm
-            * torch.sqrt(torch.tensor(x_mean.shape[2], dtype=torch.float32))
-        )
+        num_elements = x.shape[0]        
+        x_normalized: Tensor = x - x.mean(dim=0) if num_elements > 1 else x
+
+        if not self.scale:
+            return x_normalized
+        else:
+            std = torch.linalg.norm(x_normalized, dim=0) / torch.sqrt(
+                torch.tensor(x_normalized.shape[0], dtype=torch.float32)
+            )
+            assert std.dim() == x.dim() - 1
+
+            # Compute the dimensions over which we want to compute the mean standard deviation
+            dims = tuple(range(1, std.dim())) # exclude the first dimension (v)
+
+            avg_norm = std.mean(dim=dims)
+
+            # Add a singleton dimension at the beginning to allow broadcasting. 
+            # This compensates for the dimension we lost when computing the norm.
+            avg_norm = avg_norm.unsqueeze(0)
+
+            # Add singleton dimensions at the end to allow broadcasting.
+            # This compensates for the dimensions we lost when computing the mean.
+            for _ in range(1, x.dim() - 1):
+                avg_norm = avg_norm.unsqueeze(-1)
+
+            return x_normalized / avg_norm
\ No newline at end of file

From 6a20c44b8490fd64bafb925d2a311a15e43faba6 Mon Sep 17 00:00:00 2001
From: Walter Laurito <Lauritowal@yahoo.com>
Date: Fri, 28 Jul 2023 15:59:27 +0000
Subject: [PATCH 27/77] add test for burns norm

---
 tests/test_burns_norm.py | 38 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)
 create mode 100644 tests/test_burns_norm.py

diff --git a/tests/test_burns_norm.py b/tests/test_burns_norm.py
new file mode 100644
index 00000000..ac885673
--- /dev/null
+++ b/tests/test_burns_norm.py
@@ -0,0 +1,38 @@
+import torch
+from torch import Tensor
+
+from elk.training.burns_norm import BurnsNorm
+
+
+def correct_but_slow_normalization(x_all: Tensor, scale=True) -> Tensor:
+    res = []
+    xs = x_all.unbind(dim=1)
+
+    for x in xs:
+        num_elements = x.shape[0]
+        x_mean: Tensor = x - x.mean(dim=0) if num_elements > 1 else x
+        if scale == True:
+            std = torch.linalg.norm(x_mean, axis=0) / torch.sqrt(
+                torch.tensor(x_mean.shape[0], dtype=torch.float32)
+            )
+            avg_norm = std.mean()
+            x_mean = x_mean / avg_norm
+        res.append(x_mean)
+
+    return torch.stack(res, dim=1)
+
+def test_BurnsNorm_3d_input():
+    x_all_3d = torch.randn((2, 13, 768))
+    expected_output_3d = correct_but_slow_normalization(x_all_3d)
+    bn = BurnsNorm()
+    output_3d = bn(x_all_3d)
+    diff = output_3d - expected_output_3d
+    assert (diff == torch.zeros_like(diff)).all()
+
+def test_BurnsNorm_4d_input():
+    x_all_4d = torch.randn((2, 13, 2, 768))
+    expected_output_4d = correct_but_slow_normalization(x_all_4d)
+    bn = BurnsNorm()
+    output_4d = bn(x_all_4d)
+    diff = output_4d - expected_output_4d
+    assert (diff == torch.zeros_like(diff)).all()

From 8685c15b1383ec84d81e0b9aef830a4e1b162f30 Mon Sep 17 00:00:00 2001
From: Walter Laurito <Lauritowal@yahoo.com>
Date: Fri, 28 Jul 2023 16:04:57 +0000
Subject: [PATCH 28/77] no plattscale for burns norm; sigmoid instead

---
 elk/training/ccs_reporter.py | 31 ++++++++++++++++++++++---------
 elk/training/train.py        | 13 +++++++------
 2 files changed, 29 insertions(+), 15 deletions(-)

diff --git a/elk/training/ccs_reporter.py b/elk/training/ccs_reporter.py
index 01b9670c..29738dd0 100644
--- a/elk/training/ccs_reporter.py
+++ b/elk/training/ccs_reporter.py
@@ -100,14 +100,25 @@ def __init__(
 
         self.norm = None
 
-        self.probe = nn.Sequential(
-            nn.Linear(
-                in_features,
-                1 if cfg.num_layers < 2 else hidden_size,
-                bias=cfg.bias,
-                device=device,
-            ),
-        )
+        if self.cfg.norm == "burns":
+            self.probe = nn.Sequential(
+                nn.Linear(
+                    in_features,
+                    1 if cfg.num_layers < 2 else hidden_size,
+                    bias=cfg.bias,
+                    device=device,
+                ),
+                nn.Sigmoid()
+            )
+        else:
+            self.probe = nn.Sequential(
+                nn.Linear(
+                    in_features,
+                    1 if cfg.num_layers < 2 else hidden_size,
+                    bias=cfg.bias,
+                    device=device,
+                )
+            )            
         if cfg.pre_ln:
             self.probe.insert(0, nn.LayerNorm(in_features, elementwise_affine=False))
 
@@ -164,8 +175,10 @@ def reset_parameters(self):
     def forward(self, x: Tensor) -> Tensor:
         """Return the credence assigned to the hidden state `x`."""
         assert self.norm is not None, "Must call fit() before forward()"
-
+ 
         raw_scores = self.probe(self.norm(x)).squeeze(-1)
+        return raw_scores
+        breakpoint()
         return raw_scores.mul(self.scale).add(self.bias).squeeze(-1)
 
     def loss(self, logit0: Tensor, logit1: Tensor) -> Tensor:
diff --git a/elk/training/train.py b/elk/training/train.py
index 8392f2d9..014125df 100644
--- a/elk/training/train.py
+++ b/elk/training/train.py
@@ -82,12 +82,13 @@ def apply_to_layer(
 
             reporter = CcsReporter(self.net, d, device=device, num_variants=v)
             train_loss = reporter.fit(first_train_h)
-
-            (_, v, k, _) = first_train_h.shape
-            reporter.platt_scale(
-                to_one_hot(repeat(train_gt, "n -> (n v)", v=v), k).flatten(),
-                rearrange(first_train_h, "n v k d -> (n v k) d"),
-            )
+            
+            if self.net.norm == "leace":
+                (_, v, k, _) = first_train_h.shape
+                reporter.platt_scale(
+                    to_one_hot(repeat(train_gt, "n -> (n v)", v=v), k).flatten(),
+                    rearrange(first_train_h, "n v k d -> (n v k) d"),
+                )
 
         elif isinstance(self.net, EigenFitterConfig):
             fitter = EigenFitter(

From 5760f7c0d75b308aa345cd7174c119e1d682fc83 Mon Sep 17 00:00:00 2001
From: Walter Laurito <Lauritowal@yahoo.com>
Date: Fri, 28 Jul 2023 18:27:38 +0000
Subject: [PATCH 29/77] fix naming

---
 elk/training/ccs_reporter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/elk/training/ccs_reporter.py b/elk/training/ccs_reporter.py
index 29738dd0..3193001e 100644
--- a/elk/training/ccs_reporter.py
+++ b/elk/training/ccs_reporter.py
@@ -100,7 +100,7 @@ def __init__(
 
         self.norm = None
 
-        if self.cfg.norm == "burns":
+        if self.config.norm == "burns":
             self.probe = nn.Sequential(
                 nn.Linear(
                     in_features,

From 7f5a88a8b88dcb620314ce4d034d4e421df001b7 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Fri, 28 Jul 2023 18:27:53 +0000
Subject: [PATCH 30/77] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 elk/training/burns_norm.py   | 14 +++++++-------
 elk/training/ccs_reporter.py |  6 +++---
 elk/training/train.py        |  2 +-
 tests/test_burns_norm.py     |  4 +++-
 4 files changed, 14 insertions(+), 12 deletions(-)

diff --git a/elk/training/burns_norm.py b/elk/training/burns_norm.py
index c337841f..44322ed1 100644
--- a/elk/training/burns_norm.py
+++ b/elk/training/burns_norm.py
@@ -1,16 +1,16 @@
-from dataclasses import dataclass
 import torch
 from torch import Tensor, nn
 
+
 class BurnsNorm(nn.Module):
     """Burns et al. style normalization. Minimal changes from the original code."""
-    
-    def __init__(self, scale:bool=True):
+
+    def __init__(self, scale: bool = True):
         super().__init__()
         self.scale: bool = scale
 
     def forward(self, x: Tensor) -> Tensor:
-        num_elements = x.shape[0]        
+        num_elements = x.shape[0]
         x_normalized: Tensor = x - x.mean(dim=0) if num_elements > 1 else x
 
         if not self.scale:
@@ -22,11 +22,11 @@ def forward(self, x: Tensor) -> Tensor:
             assert std.dim() == x.dim() - 1
 
             # Compute the dimensions over which we want to compute the mean standard deviation
-            dims = tuple(range(1, std.dim())) # exclude the first dimension (v)
+            dims = tuple(range(1, std.dim()))  # exclude the first dimension (v)
 
             avg_norm = std.mean(dim=dims)
 
-            # Add a singleton dimension at the beginning to allow broadcasting. 
+            # Add a singleton dimension at the beginning to allow broadcasting.
             # This compensates for the dimension we lost when computing the norm.
             avg_norm = avg_norm.unsqueeze(0)
 
@@ -35,4 +35,4 @@ def forward(self, x: Tensor) -> Tensor:
             for _ in range(1, x.dim() - 1):
                 avg_norm = avg_norm.unsqueeze(-1)
 
-            return x_normalized / avg_norm
\ No newline at end of file
+            return x_normalized / avg_norm
diff --git a/elk/training/ccs_reporter.py b/elk/training/ccs_reporter.py
index 3193001e..76a93236 100644
--- a/elk/training/ccs_reporter.py
+++ b/elk/training/ccs_reporter.py
@@ -108,7 +108,7 @@ def __init__(
                     bias=cfg.bias,
                     device=device,
                 ),
-                nn.Sigmoid()
+                nn.Sigmoid(),
             )
         else:
             self.probe = nn.Sequential(
@@ -118,7 +118,7 @@ def __init__(
                     bias=cfg.bias,
                     device=device,
                 )
-            )            
+            )
         if cfg.pre_ln:
             self.probe.insert(0, nn.LayerNorm(in_features, elementwise_affine=False))
 
@@ -175,7 +175,7 @@ def reset_parameters(self):
     def forward(self, x: Tensor) -> Tensor:
         """Return the credence assigned to the hidden state `x`."""
         assert self.norm is not None, "Must call fit() before forward()"
- 
+
         raw_scores = self.probe(self.norm(x)).squeeze(-1)
         return raw_scores
         breakpoint()
diff --git a/elk/training/train.py b/elk/training/train.py
index 822aa2c4..b6888ea7 100644
--- a/elk/training/train.py
+++ b/elk/training/train.py
@@ -82,7 +82,7 @@ def apply_to_layer(
 
             reporter = CcsReporter(self.net, d, device=device, num_variants=v)
             train_loss = reporter.fit(first_train_h)
-            
+
             if self.net.norm == "leace":
                 (_, v, k, _) = first_train_h.shape
                 reporter.platt_scale(
diff --git a/tests/test_burns_norm.py b/tests/test_burns_norm.py
index ac885673..4620ae0e 100644
--- a/tests/test_burns_norm.py
+++ b/tests/test_burns_norm.py
@@ -11,7 +11,7 @@ def correct_but_slow_normalization(x_all: Tensor, scale=True) -> Tensor:
     for x in xs:
         num_elements = x.shape[0]
         x_mean: Tensor = x - x.mean(dim=0) if num_elements > 1 else x
-        if scale == True:
+        if scale is True:
             std = torch.linalg.norm(x_mean, axis=0) / torch.sqrt(
                 torch.tensor(x_mean.shape[0], dtype=torch.float32)
             )
@@ -21,6 +21,7 @@ def correct_but_slow_normalization(x_all: Tensor, scale=True) -> Tensor:
 
     return torch.stack(res, dim=1)
 
+
 def test_BurnsNorm_3d_input():
     x_all_3d = torch.randn((2, 13, 768))
     expected_output_3d = correct_but_slow_normalization(x_all_3d)
@@ -29,6 +30,7 @@ def test_BurnsNorm_3d_input():
     diff = output_3d - expected_output_3d
     assert (diff == torch.zeros_like(diff)).all()
 
+
 def test_BurnsNorm_4d_input():
     x_all_4d = torch.randn((2, 13, 2, 768))
     expected_output_4d = correct_but_slow_normalization(x_all_4d)

From b15f55e94c4d0d5da0efeeb098d1dcdbba524315 Mon Sep 17 00:00:00 2001
From: Walter Laurito <Lauritowal@yahoo.com>
Date: Fri, 28 Jul 2023 18:49:58 +0000
Subject: [PATCH 31/77] pre-commit cleanup

---
 elk/training/burns_norm.py   | 18 ++++++++++--------
 elk/training/ccs_reporter.py |  6 +++---
 elk/training/train.py        |  2 +-
 tests/test_burns_norm.py     |  4 +++-
 4 files changed, 17 insertions(+), 13 deletions(-)

diff --git a/elk/training/burns_norm.py b/elk/training/burns_norm.py
index c337841f..e88616ba 100644
--- a/elk/training/burns_norm.py
+++ b/elk/training/burns_norm.py
@@ -1,16 +1,16 @@
-from dataclasses import dataclass
 import torch
 from torch import Tensor, nn
 
+
 class BurnsNorm(nn.Module):
     """Burns et al. style normalization. Minimal changes from the original code."""
-    
-    def __init__(self, scale:bool=True):
+
+    def __init__(self, scale: bool = True):
         super().__init__()
         self.scale: bool = scale
 
     def forward(self, x: Tensor) -> Tensor:
-        num_elements = x.shape[0]        
+        num_elements = x.shape[0]
         x_normalized: Tensor = x - x.mean(dim=0) if num_elements > 1 else x
 
         if not self.scale:
@@ -21,12 +21,14 @@ def forward(self, x: Tensor) -> Tensor:
             )
             assert std.dim() == x.dim() - 1
 
-            # Compute the dimensions over which we want to compute the mean standard deviation
-            dims = tuple(range(1, std.dim())) # exclude the first dimension (v)
+            # Compute the dimensions over which
+            # we want to compute the mean standard deviation
+            # exclude the first dimension (v)
+            dims = tuple(range(1, std.dim()))
 
             avg_norm = std.mean(dim=dims)
 
-            # Add a singleton dimension at the beginning to allow broadcasting. 
+            # Add a singleton dimension at the beginning to allow broadcasting.
             # This compensates for the dimension we lost when computing the norm.
             avg_norm = avg_norm.unsqueeze(0)
 
@@ -35,4 +37,4 @@ def forward(self, x: Tensor) -> Tensor:
             for _ in range(1, x.dim() - 1):
                 avg_norm = avg_norm.unsqueeze(-1)
 
-            return x_normalized / avg_norm
\ No newline at end of file
+            return x_normalized / avg_norm
diff --git a/elk/training/ccs_reporter.py b/elk/training/ccs_reporter.py
index 3193001e..76a93236 100644
--- a/elk/training/ccs_reporter.py
+++ b/elk/training/ccs_reporter.py
@@ -108,7 +108,7 @@ def __init__(
                     bias=cfg.bias,
                     device=device,
                 ),
-                nn.Sigmoid()
+                nn.Sigmoid(),
             )
         else:
             self.probe = nn.Sequential(
@@ -118,7 +118,7 @@ def __init__(
                     bias=cfg.bias,
                     device=device,
                 )
-            )            
+            )
         if cfg.pre_ln:
             self.probe.insert(0, nn.LayerNorm(in_features, elementwise_affine=False))
 
@@ -175,7 +175,7 @@ def reset_parameters(self):
     def forward(self, x: Tensor) -> Tensor:
         """Return the credence assigned to the hidden state `x`."""
         assert self.norm is not None, "Must call fit() before forward()"
- 
+
         raw_scores = self.probe(self.norm(x)).squeeze(-1)
         return raw_scores
         breakpoint()
diff --git a/elk/training/train.py b/elk/training/train.py
index 822aa2c4..b6888ea7 100644
--- a/elk/training/train.py
+++ b/elk/training/train.py
@@ -82,7 +82,7 @@ def apply_to_layer(
 
             reporter = CcsReporter(self.net, d, device=device, num_variants=v)
             train_loss = reporter.fit(first_train_h)
-            
+
             if self.net.norm == "leace":
                 (_, v, k, _) = first_train_h.shape
                 reporter.platt_scale(
diff --git a/tests/test_burns_norm.py b/tests/test_burns_norm.py
index ac885673..4620ae0e 100644
--- a/tests/test_burns_norm.py
+++ b/tests/test_burns_norm.py
@@ -11,7 +11,7 @@ def correct_but_slow_normalization(x_all: Tensor, scale=True) -> Tensor:
     for x in xs:
         num_elements = x.shape[0]
         x_mean: Tensor = x - x.mean(dim=0) if num_elements > 1 else x
-        if scale == True:
+        if scale is True:
             std = torch.linalg.norm(x_mean, axis=0) / torch.sqrt(
                 torch.tensor(x_mean.shape[0], dtype=torch.float32)
             )
@@ -21,6 +21,7 @@ def correct_but_slow_normalization(x_all: Tensor, scale=True) -> Tensor:
 
     return torch.stack(res, dim=1)
 
+
 def test_BurnsNorm_3d_input():
     x_all_3d = torch.randn((2, 13, 768))
     expected_output_3d = correct_but_slow_normalization(x_all_3d)
@@ -29,6 +30,7 @@ def test_BurnsNorm_3d_input():
     diff = output_3d - expected_output_3d
     assert (diff == torch.zeros_like(diff)).all()
 
+
 def test_BurnsNorm_4d_input():
     x_all_4d = torch.randn((2, 13, 2, 768))
     expected_output_4d = correct_but_slow_normalization(x_all_4d)

From 02c2a9ca0de8a238565b306d0d3a35c9807e63f0 Mon Sep 17 00:00:00 2001
From: Walter Laurito <Lauritowal@yahoo.com>
Date: Fri, 28 Jul 2023 18:56:41 +0000
Subject: [PATCH 32/77] remove code duplication

---
 elk/training/ccs_reporter.py | 29 ++++++++++-------------------
 1 file changed, 10 insertions(+), 19 deletions(-)

diff --git a/elk/training/ccs_reporter.py b/elk/training/ccs_reporter.py
index 76a93236..d603be1f 100644
--- a/elk/training/ccs_reporter.py
+++ b/elk/training/ccs_reporter.py
@@ -100,25 +100,16 @@ def __init__(
 
         self.norm = None
 
-        if self.config.norm == "burns":
-            self.probe = nn.Sequential(
-                nn.Linear(
-                    in_features,
-                    1 if cfg.num_layers < 2 else hidden_size,
-                    bias=cfg.bias,
-                    device=device,
-                ),
-                nn.Sigmoid(),
-            )
-        else:
-            self.probe = nn.Sequential(
-                nn.Linear(
-                    in_features,
-                    1 if cfg.num_layers < 2 else hidden_size,
-                    bias=cfg.bias,
-                    device=device,
-                )
-            )
+        self.probe = nn.Sequential(
+            nn.Linear(
+                in_features,
+                1 if cfg.num_layers < 2 else hidden_size,
+                bias=cfg.bias,
+                device=device,
+            ),
+            *(nn.Sigmoid() if self.config.norm == "burns" else ()),
+        )
+
         if cfg.pre_ln:
             self.probe.insert(0, nn.LayerNorm(in_features, elementwise_affine=False))
 

From e172ab63ba80347446f03c949fbbfe818fe3c1f1 Mon Sep 17 00:00:00 2001
From: Walter Laurito <Lauritowal@yahoo.com>
Date: Fri, 28 Jul 2023 19:04:23 +0000
Subject: [PATCH 33/77] cleanup code duplication

---
 elk/training/ccs_reporter.py | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/elk/training/ccs_reporter.py b/elk/training/ccs_reporter.py
index d603be1f..03ede336 100644
--- a/elk/training/ccs_reporter.py
+++ b/elk/training/ccs_reporter.py
@@ -100,16 +100,18 @@ def __init__(
 
         self.norm = None
 
-        self.probe = nn.Sequential(
-            nn.Linear(
-                in_features,
-                1 if cfg.num_layers < 2 else hidden_size,
-                bias=cfg.bias,
-                device=device,
-            ),
-            *(nn.Sigmoid() if self.config.norm == "burns" else ()),
-        )
+        layers = [nn.Linear(
+            in_features,
+            1 if cfg.num_layers < 2 else hidden_size,
+            bias=cfg.bias,
+            device=device,
+        )]
+
+        if self.config.norm == "burns":
+            layers.append(nn.Sigmoid())
 
+        self.probe = nn.Sequential(*layers)
+        
         if cfg.pre_ln:
             self.probe.insert(0, nn.LayerNorm(in_features, elementwise_affine=False))
 

From 76b5eb1617cfcdafbcde124b0f1594a18e6817fe Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Fri, 28 Jul 2023 19:04:39 +0000
Subject: [PATCH 34/77] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 elk/training/ccs_reporter.py | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/elk/training/ccs_reporter.py b/elk/training/ccs_reporter.py
index 03ede336..6f9257f0 100644
--- a/elk/training/ccs_reporter.py
+++ b/elk/training/ccs_reporter.py
@@ -100,18 +100,20 @@ def __init__(
 
         self.norm = None
 
-        layers = [nn.Linear(
-            in_features,
-            1 if cfg.num_layers < 2 else hidden_size,
-            bias=cfg.bias,
-            device=device,
-        )]
+        layers = [
+            nn.Linear(
+                in_features,
+                1 if cfg.num_layers < 2 else hidden_size,
+                bias=cfg.bias,
+                device=device,
+            )
+        ]
 
         if self.config.norm == "burns":
             layers.append(nn.Sigmoid())
 
         self.probe = nn.Sequential(*layers)
-        
+
         if cfg.pre_ln:
             self.probe.insert(0, nn.LayerNorm(in_features, elementwise_affine=False))
 

From 96e05608439172f5946810a44460582bb8600fe3 Mon Sep 17 00:00:00 2001
From: Walter Laurito <Lauritowal@yahoo.com>
Date: Fri, 28 Jul 2023 19:15:37 +0000
Subject: [PATCH 35/77] add type hint

---
 elk/training/ccs_reporter.py | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/elk/training/ccs_reporter.py b/elk/training/ccs_reporter.py
index 03ede336..966a45e5 100644
--- a/elk/training/ccs_reporter.py
+++ b/elk/training/ccs_reporter.py
@@ -100,18 +100,20 @@ def __init__(
 
         self.norm = None
 
-        layers = [nn.Linear(
-            in_features,
-            1 if cfg.num_layers < 2 else hidden_size,
-            bias=cfg.bias,
-            device=device,
-        )]
+        layers: list[nn.Module] = [
+            nn.Linear(
+                in_features,
+                1 if cfg.num_layers < 2 else hidden_size,
+                bias=cfg.bias,
+                device=device,
+            )
+        ]
 
         if self.config.norm == "burns":
             layers.append(nn.Sigmoid())
 
         self.probe = nn.Sequential(*layers)
-        
+
         if cfg.pre_ln:
             self.probe.insert(0, nn.LayerNorm(in_features, elementwise_affine=False))
 

From 001deb83dbf8846d5693ec14ec22ac5b031521c6 Mon Sep 17 00:00:00 2001
From: Walter Laurito <Lauritowal@yahoo.com>
Date: Fri, 28 Jul 2023 19:19:16 +0000
Subject: [PATCH 36/77] fix forward by checking for leace

---
 elk/training/ccs_reporter.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/elk/training/ccs_reporter.py b/elk/training/ccs_reporter.py
index 6f9257f0..25bd17d3 100644
--- a/elk/training/ccs_reporter.py
+++ b/elk/training/ccs_reporter.py
@@ -172,9 +172,10 @@ def forward(self, x: Tensor) -> Tensor:
         assert self.norm is not None, "Must call fit() before forward()"
 
         raw_scores = self.probe(self.norm(x)).squeeze(-1)
-        return raw_scores
-        breakpoint()
-        return raw_scores.mul(self.scale).add(self.bias).squeeze(-1)
+        if self.config.norm == "leace":
+            return raw_scores.mul(self.scale).add(self.bias).squeeze(-1)
+        else:
+            return raw_scores
 
     def loss(self, logit0: Tensor, logit1: Tensor) -> Tensor:
         """Return the loss of the reporter on the contrast pair (x0, x1).

From e8872c8a28f2b5da878af840560b7533ec0d7b80 Mon Sep 17 00:00:00 2001
From: Walter Laurito <Lauritowal@yahoo.com>
Date: Fri, 28 Jul 2023 19:36:22 +0000
Subject: [PATCH 37/77] add annotation

---
 elk/training/burns_norm.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/elk/training/burns_norm.py b/elk/training/burns_norm.py
index e88616ba..331964c9 100644
--- a/elk/training/burns_norm.py
+++ b/elk/training/burns_norm.py
@@ -3,13 +3,20 @@
 
 
 class BurnsNorm(nn.Module):
-    """Burns et al. style normalization. Minimal changes from the original code."""
+    """Burns et al. style normalization. Minimal changes from the original code.
+    """
 
     def __init__(self, scale: bool = True):
         super().__init__()
         self.scale: bool = scale
 
     def forward(self, x: Tensor) -> Tensor:
+        """Normalizes per template
+        Args: 
+            x: input of dimension (n, v, c, d) or (n, v, d)
+        Returns:
+            x_normalized: normalized output
+        """
         num_elements = x.shape[0]
         x_normalized: Tensor = x - x.mean(dim=0) if num_elements > 1 else x
 
@@ -23,7 +30,8 @@ def forward(self, x: Tensor) -> Tensor:
 
             # Compute the dimensions over which
             # we want to compute the mean standard deviation
-            # exclude the first dimension (v)
+            # exclude the first dimension v, 
+            # which is the template dimension
             dims = tuple(range(1, std.dim()))
 
             avg_norm = std.mean(dim=dims)

From 0f35b96a6cc5b4c24fcf6e01d36a2f1e7be36f21 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Fri, 28 Jul 2023 19:40:46 +0000
Subject: [PATCH 38/77] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 elk/training/burns_norm.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/elk/training/burns_norm.py b/elk/training/burns_norm.py
index 331964c9..f9925f2f 100644
--- a/elk/training/burns_norm.py
+++ b/elk/training/burns_norm.py
@@ -3,8 +3,7 @@
 
 
 class BurnsNorm(nn.Module):
-    """Burns et al. style normalization. Minimal changes from the original code.
-    """
+    """Burns et al. style normalization. Minimal changes from the original code."""
 
     def __init__(self, scale: bool = True):
         super().__init__()
@@ -12,7 +11,7 @@ def __init__(self, scale: bool = True):
 
     def forward(self, x: Tensor) -> Tensor:
         """Normalizes per template
-        Args: 
+        Args:
             x: input of dimension (n, v, c, d) or (n, v, d)
         Returns:
             x_normalized: normalized output
@@ -30,7 +29,7 @@ def forward(self, x: Tensor) -> Tensor:
 
             # Compute the dimensions over which
             # we want to compute the mean standard deviation
-            # exclude the first dimension v, 
+            # exclude the first dimension v,
             # which is the template dimension
             dims = tuple(range(1, std.dim()))
 

From 94a498e2dc0d2dc3307aad27329e6517c1bfb166 Mon Sep 17 00:00:00 2001
From: Walter Laurito <Lauritowal@yahoo.com>
Date: Fri, 28 Jul 2023 19:42:46 +0000
Subject: [PATCH 39/77] add typing

---
 elk/training/ccs_reporter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/elk/training/ccs_reporter.py b/elk/training/ccs_reporter.py
index 25bd17d3..89b19f6b 100644
--- a/elk/training/ccs_reporter.py
+++ b/elk/training/ccs_reporter.py
@@ -100,7 +100,7 @@ def __init__(
 
         self.norm = None
 
-        layers = [
+        layers: list[nn.Module] = [
             nn.Linear(
                 in_features,
                 1 if cfg.num_layers < 2 else hidden_size,

From 99646e9a85b770d98a29af958576627dde638b4e Mon Sep 17 00:00:00 2001
From: Walter Laurito <Lauritowal@yahoo.com>
Date: Fri, 28 Jul 2023 19:52:14 +0000
Subject: [PATCH 40/77] rename to probe_layers

---
 elk/training/ccs_reporter.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/elk/training/ccs_reporter.py b/elk/training/ccs_reporter.py
index 89b19f6b..525c1490 100644
--- a/elk/training/ccs_reporter.py
+++ b/elk/training/ccs_reporter.py
@@ -100,7 +100,7 @@ def __init__(
 
         self.norm = None
 
-        layers: list[nn.Module] = [
+        probe_layers: list[nn.Module] = [
             nn.Linear(
                 in_features,
                 1 if cfg.num_layers < 2 else hidden_size,
@@ -110,9 +110,9 @@ def __init__(
         ]
 
         if self.config.norm == "burns":
-            layers.append(nn.Sigmoid())
+            probe_layers.append(nn.Sigmoid())
 
-        self.probe = nn.Sequential(*layers)
+        self.probe = nn.Sequential(*probe_layers)
 
         if cfg.pre_ln:
             self.probe.insert(0, nn.LayerNorm(in_features, elementwise_affine=False))

From aeb8b099e98f9dc52d5c02ec0d092f59c610a28d Mon Sep 17 00:00:00 2001
From: Walter Laurito <lauritowal@yahoo.com>
Date: Thu, 3 Aug 2023 17:47:38 +0200
Subject: [PATCH 41/77] remove sigmoid from ccs reporter class + cleanup

---
 elk/training/ccs_reporter.py | 17 +++++++----------
 elk/training/train.py        |  2 +-
 2 files changed, 8 insertions(+), 11 deletions(-)

diff --git a/elk/training/ccs_reporter.py b/elk/training/ccs_reporter.py
index 525c1490..f6cc25ff 100644
--- a/elk/training/ccs_reporter.py
+++ b/elk/training/ccs_reporter.py
@@ -99,20 +99,14 @@ def __init__(
         hidden_size = cfg.hidden_size or 4 * in_features // 3
 
         self.norm = None
-
-        probe_layers: list[nn.Module] = [
+        self.probe: nn.Sequential(
             nn.Linear(
                 in_features,
                 1 if cfg.num_layers < 2 else hidden_size,
                 bias=cfg.bias,
                 device=device,
-            )
-        ]
-
-        if self.config.norm == "burns":
-            probe_layers.append(nn.Sigmoid())
-
-        self.probe = nn.Sequential(*probe_layers)
+            ),
+        )
 
         if cfg.pre_ln:
             self.probe.insert(0, nn.LayerNorm(in_features, elementwise_affine=False))
@@ -174,8 +168,11 @@ def forward(self, x: Tensor) -> Tensor:
         raw_scores = self.probe(self.norm(x)).squeeze(-1)
         if self.config.norm == "leace":
             return raw_scores.mul(self.scale).add(self.bias).squeeze(-1)
-        else:
+
+        elif self.config.norm == "burns":
             return raw_scores
+        else:
+            raise ValueError(f"Unknown normalization {self.config.norm}.")
 
     def loss(self, logit0: Tensor, logit1: Tensor) -> Tensor:
         """Return the loss of the reporter on the contrast pair (x0, x1).
diff --git a/elk/training/train.py b/elk/training/train.py
index b6888ea7..a7f0ef07 100644
--- a/elk/training/train.py
+++ b/elk/training/train.py
@@ -83,7 +83,7 @@ def apply_to_layer(
             reporter = CcsReporter(self.net, d, device=device, num_variants=v)
             train_loss = reporter.fit(first_train_h)
 
-            if self.net.norm == "leace":
+            if not self.net.norm == "burns":
                 (_, v, k, _) = first_train_h.shape
                 reporter.platt_scale(
                     to_one_hot(repeat(train_gt, "n -> (n v)", v=v), k).flatten(),

From f2ae1b7bf1cbc3a359231f1084a034f0be8f1f2b Mon Sep 17 00:00:00 2001
From: Walter Laurito <lauritowal@yahoo.com>
Date: Thu, 3 Aug 2023 17:53:41 +0200
Subject: [PATCH 42/77] fix assignment of probe

---
 elk/training/ccs_reporter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/elk/training/ccs_reporter.py b/elk/training/ccs_reporter.py
index f6cc25ff..472417f5 100644
--- a/elk/training/ccs_reporter.py
+++ b/elk/training/ccs_reporter.py
@@ -99,7 +99,7 @@ def __init__(
         hidden_size = cfg.hidden_size or 4 * in_features // 3
 
         self.norm = None
-        self.probe: nn.Sequential(
+        self.probe = nn.Sequential(
             nn.Linear(
                 in_features,
                 1 if cfg.num_layers < 2 else hidden_size,

From d10b4c2b888b569648f67bf2e460a78dcd772d6b Mon Sep 17 00:00:00 2001
From: Walter Laurito <Lauritowal@yahoo.com>
Date: Sat, 5 Aug 2023 22:19:04 +0100
Subject: [PATCH 43/77] remove calling self.leace.update twice (#283)

---
 elk/training/eigen_reporter.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/elk/training/eigen_reporter.py b/elk/training/eigen_reporter.py
index a3525b1d..68fd89ed 100644
--- a/elk/training/eigen_reporter.py
+++ b/elk/training/eigen_reporter.py
@@ -147,7 +147,6 @@ def update(self, hiddens: Tensor) -> None:
         if self.config.erase_prompts:
             # Independent indicator for each (template, pseudo-label) pair
             indicators = torch.eye(k * v, device=hiddens.device).expand(n, -1, -1)
-            self.leace.update(x=hiddens, z=indicators)
         else:
             # Only use indicators for each pseudo-label
             indicators = torch.eye(k, device=hiddens.device).expand(n, v, -1, -1)

From 3d6042f13fde4c9791dc88d8f86cb7d6d24dfefa Mon Sep 17 00:00:00 2001
From: Walter Laurito <Lauritowal@yahoo.com>
Date: Fri, 11 Aug 2023 11:08:20 +0000
Subject: [PATCH 44/77] remove unnecessary tensor wrapping

---
 comparison-sweeps          | 1 +
 elk/training/burns_norm.py | 4 +---
 2 files changed, 2 insertions(+), 3 deletions(-)
 create mode 160000 comparison-sweeps

diff --git a/comparison-sweeps b/comparison-sweeps
new file mode 160000
index 00000000..f4ed884b
--- /dev/null
+++ b/comparison-sweeps
@@ -0,0 +1 @@
+Subproject commit f4ed884b59c99012c80b972d2a02c660b39c90cb
diff --git a/elk/training/burns_norm.py b/elk/training/burns_norm.py
index f9925f2f..8c87efe0 100644
--- a/elk/training/burns_norm.py
+++ b/elk/training/burns_norm.py
@@ -22,9 +22,7 @@ def forward(self, x: Tensor) -> Tensor:
         if not self.scale:
             return x_normalized
         else:
-            std = torch.linalg.norm(x_normalized, dim=0) / torch.sqrt(
-                torch.tensor(x_normalized.shape[0], dtype=torch.float32)
-            )
+            std = torch.linalg.norm(x_normalized, dim=0) / x_normalized.shape[0] ** 0.5
             assert std.dim() == x.dim() - 1
 
             # Compute the dimensions over which

From ee090c810fc8735aa460ca08334fa0b10f8fb82a Mon Sep 17 00:00:00 2001
From: Walter Laurito <Lauritowal@yahoo.com>
Date: Fri, 11 Aug 2023 11:19:27 +0000
Subject: [PATCH 45/77] remove unnessary unsqueezes; using keepdim instead

---
 elk/training/burns_norm.py | 11 +----------
 1 file changed, 1 insertion(+), 10 deletions(-)

diff --git a/elk/training/burns_norm.py b/elk/training/burns_norm.py
index 8c87efe0..5a0adc69 100644
--- a/elk/training/burns_norm.py
+++ b/elk/training/burns_norm.py
@@ -31,15 +31,6 @@ def forward(self, x: Tensor) -> Tensor:
             # which is the template dimension
             dims = tuple(range(1, std.dim()))
 
-            avg_norm = std.mean(dim=dims)
-
-            # Add a singleton dimension at the beginning to allow broadcasting.
-            # This compensates for the dimension we lost when computing the norm.
-            avg_norm = avg_norm.unsqueeze(0)
-
-            # Add singleton dimensions at the end to allow broadcasting.
-            # This compensates for the dimensions we lost when computing the mean.
-            for _ in range(1, x.dim() - 1):
-                avg_norm = avg_norm.unsqueeze(-1)
+            avg_norm = std.mean(dim=dims, keepdim=True)
 
             return x_normalized / avg_norm

From d6b5ce6d157d8e48f31235a5d034918e15fcd2fd Mon Sep 17 00:00:00 2001
From: Walter Laurito <Lauritowal@yahoo.com>
Date: Fri, 11 Aug 2023 11:22:20 +0000
Subject: [PATCH 46/77] clearer commment for what template is used

---
 elk/training/burns_norm.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/elk/training/burns_norm.py b/elk/training/burns_norm.py
index 5a0adc69..f116937e 100644
--- a/elk/training/burns_norm.py
+++ b/elk/training/burns_norm.py
@@ -10,7 +10,7 @@ def __init__(self, scale: bool = True):
         self.scale: bool = scale
 
     def forward(self, x: Tensor) -> Tensor:
-        """Normalizes per template
+        """Normalizes per prompt template
         Args:
             x: input of dimension (n, v, c, d) or (n, v, d)
         Returns:

From 8756fa4332bc2a03b8def6dcaea76fbb12508625 Mon Sep 17 00:00:00 2001
From: Walter Laurito <Lauritowal@yahoo.com>
Date: Fri, 11 Aug 2023 20:49:08 +0000
Subject: [PATCH 47/77] remove first layer (embeddings) from layer_indices

---
 elk/extraction/extraction.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/elk/extraction/extraction.py b/elk/extraction/extraction.py
index 26e8a7f1..aad05ff7 100644
--- a/elk/extraction/extraction.py
+++ b/elk/extraction/extraction.py
@@ -194,7 +194,8 @@ def extract_hiddens(
         seed=cfg.seed,
     )
 
-    layer_indices = cfg.layers or tuple(range(model.config.num_hidden_layers))
+    layer_indices = cfg.layers or tuple(range(model.config.num_hidden_layers)[1:])
+
 
     global_max_examples = cfg.max_examples[0 if split_type == "train" else 1]
 
@@ -366,12 +367,13 @@ def hidden_features(cfg: Extract) -> tuple[DatasetInfo, Features]:
     if num_dropped:
         print(f"Dropping {num_dropped} non-multiple choice templates")
 
+    layer_indices = cfg.layers or tuple(range(model_cfg.num_hidden_layers)[1:])
     layer_cols = {
         f"hidden_{layer}": Array3D(
             dtype="int16",
             shape=(num_variants, num_classes, model_cfg.hidden_size),
         )
-        for layer in cfg.layers or range(model_cfg.num_hidden_layers)
+        for layer in layer_indices
     }
     other_cols = {
         "variant_ids": Sequence(
@@ -458,6 +460,7 @@ def extract(
     mp.set_start_method("spawn", force=True)  # type: ignore[attr-defined]
 
     ds = dict()
+    breakpoint()
     for split, builder in builders.items():
         builder.download_and_prepare(
             download_mode=DownloadMode.FORCE_REDOWNLOAD if disable_cache else None,

From 015b5f523dc7b3ba90dbe15b156e60a350d2bd7e Mon Sep 17 00:00:00 2001
From: Walter Laurito <Lauritowal@yahoo.com>
Date: Fri, 11 Aug 2023 21:17:37 +0000
Subject: [PATCH 48/77] remove breakpoint

---
 elk/extraction/extraction.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/elk/extraction/extraction.py b/elk/extraction/extraction.py
index aad05ff7..5f6db432 100644
--- a/elk/extraction/extraction.py
+++ b/elk/extraction/extraction.py
@@ -460,7 +460,6 @@ def extract(
     mp.set_start_method("spawn", force=True)  # type: ignore[attr-defined]
 
     ds = dict()
-    breakpoint()
     for split, builder in builders.items():
         builder.download_and_prepare(
             download_mode=DownloadMode.FORCE_REDOWNLOAD if disable_cache else None,

From cc095cb5e2a574b3cce82933258fdba63daf7151 Mon Sep 17 00:00:00 2001
From: Walter Laurito <Lauritowal@yahoo.com>
Date: Fri, 11 Aug 2023 21:18:51 +0000
Subject: [PATCH 49/77] pre-commit cleanup

---
 elk/extraction/extraction.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/elk/extraction/extraction.py b/elk/extraction/extraction.py
index 5f6db432..30bb0adc 100644
--- a/elk/extraction/extraction.py
+++ b/elk/extraction/extraction.py
@@ -196,7 +196,6 @@ def extract_hiddens(
 
     layer_indices = cfg.layers or tuple(range(model.config.num_hidden_layers)[1:])
 
-
     global_max_examples = cfg.max_examples[0 if split_type == "train" else 1]
 
     # break `max_examples` among the processes roughly equally

From af4c9d11b710c8c60c832d4ce8ff6a7f5607abe4 Mon Sep 17 00:00:00 2001
From: Walter Laurito <Lauritowal@yahoo.com>
Date: Fri, 11 Aug 2023 21:50:21 +0000
Subject: [PATCH 50/77] replace slicing

---
 elk/extraction/extraction.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/elk/extraction/extraction.py b/elk/extraction/extraction.py
index 30bb0adc..fe4fdc84 100644
--- a/elk/extraction/extraction.py
+++ b/elk/extraction/extraction.py
@@ -194,7 +194,7 @@ def extract_hiddens(
         seed=cfg.seed,
     )
 
-    layer_indices = cfg.layers or tuple(range(model.config.num_hidden_layers)[1:])
+    layer_indices = cfg.layers or tuple(range(1, model.config.num_hidden_layers))
 
     global_max_examples = cfg.max_examples[0 if split_type == "train" else 1]
 
@@ -366,7 +366,7 @@ def hidden_features(cfg: Extract) -> tuple[DatasetInfo, Features]:
     if num_dropped:
         print(f"Dropping {num_dropped} non-multiple choice templates")
 
-    layer_indices = cfg.layers or tuple(range(model_cfg.num_hidden_layers)[1:])
+    layer_indices = cfg.layers or tuple(range(1, model_cfg.num_hidden_layers))
     layer_cols = {
         f"hidden_{layer}": Array3D(
             dtype="int16",

From 5d4e7a414c4a2f629f6c298a09de4d3bef5b3496 Mon Sep 17 00:00:00 2001
From: Walter Laurito <Lauritowal@yahoo.com>
Date: Wed, 23 Aug 2023 13:01:22 +0200
Subject: [PATCH 51/77] add platt scaling for burns + fix for leace

---
 elk/training/ccs_reporter.py | 13 +++++++------
 elk/training/train.py        |  9 +++------
 2 files changed, 10 insertions(+), 12 deletions(-)

diff --git a/elk/training/ccs_reporter.py b/elk/training/ccs_reporter.py
index 472417f5..cf9325f6 100644
--- a/elk/training/ccs_reporter.py
+++ b/elk/training/ccs_reporter.py
@@ -88,6 +88,8 @@ def __init__(
         num_variants: int = 1,
     ):
         super().__init__()
+        self._is_training = True
+
         self.config = cfg
         self.in_features = in_features
         self.num_variants = num_variants
@@ -164,15 +166,12 @@ def reset_parameters(self):
     def forward(self, x: Tensor) -> Tensor:
         """Return the credence assigned to the hidden state `x`."""
         assert self.norm is not None, "Must call fit() before forward()"
-
         raw_scores = self.probe(self.norm(x)).squeeze(-1)
-        if self.config.norm == "leace":
-            return raw_scores.mul(self.scale).add(self.bias).squeeze(-1)
-
-        elif self.config.norm == "burns":
+        if self._is_training:
             return raw_scores
         else:
-            raise ValueError(f"Unknown normalization {self.config.norm}.")
+            platt_scaled_scores = raw_scores.mul(self.scale).add(self.bias).squeeze(-1)
+            return platt_scaled_scores
 
     def loss(self, logit0: Tensor, logit1: Tensor) -> Tensor:
         """Return the loss of the reporter on the contrast pair (x0, x1).
@@ -248,6 +247,8 @@ def fit(self, hiddens: Tensor) -> float:
             raise RuntimeError("Got NaN/infinite loss during training")
 
         self.load_state_dict(best_state)
+
+        self._is_training = False
         return best_loss
 
     def train_loop_adam(self, x_neg: Tensor, x_pos: Tensor) -> float:
diff --git a/elk/training/train.py b/elk/training/train.py
index a7f0ef07..cc2d0fbc 100644
--- a/elk/training/train.py
+++ b/elk/training/train.py
@@ -83,12 +83,9 @@ def apply_to_layer(
             reporter = CcsReporter(self.net, d, device=device, num_variants=v)
             train_loss = reporter.fit(first_train_h)
 
-            if not self.net.norm == "burns":
-                (_, v, k, _) = first_train_h.shape
-                reporter.platt_scale(
-                    to_one_hot(repeat(train_gt, "n -> (n v)", v=v), k).flatten(),
-                    rearrange(first_train_h, "n v k d -> (n v k) d"),
-                )
+            labels = to_one_hot(train_gt, k)
+            labels = repeat(train_gt, "n -> n v k", v=v, k=k)
+            reporter.platt_scale(labels, first_train_h)
 
         elif isinstance(self.net, EigenFitterConfig):
             fitter = EigenFitter(

From 707ce8e87f2f5f57c714b274d98cb1cc72eed2bb Mon Sep 17 00:00:00 2001
From: Walter Laurito <Lauritowal@yahoo.com>
Date: Wed, 23 Aug 2023 17:23:52 +0200
Subject: [PATCH 52/77] add comment

---
 elk/training/ccs_reporter.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/elk/training/ccs_reporter.py b/elk/training/ccs_reporter.py
index cf9325f6..1c65d98d 100644
--- a/elk/training/ccs_reporter.py
+++ b/elk/training/ccs_reporter.py
@@ -170,6 +170,7 @@ def forward(self, x: Tensor) -> Tensor:
         if self._is_training:
             return raw_scores
         else:
+            # only do platt scaling after training the reporters
             platt_scaled_scores = raw_scores.mul(self.scale).add(self.bias).squeeze(-1)
             return platt_scaled_scores
 

From 777912ea1983966ca0b3c585bdad4a2291b5418a Mon Sep 17 00:00:00 2001
From: Walter Laurito <Lauritowal@yahoo.com>
Date: Wed, 23 Aug 2023 22:23:00 +0200
Subject: [PATCH 53/77] repeat onehot in template dimension

---
 elk/training/train.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/elk/training/train.py b/elk/training/train.py
index cc2d0fbc..84e81ccc 100644
--- a/elk/training/train.py
+++ b/elk/training/train.py
@@ -82,10 +82,8 @@ def apply_to_layer(
 
             reporter = CcsReporter(self.net, d, device=device, num_variants=v)
             train_loss = reporter.fit(first_train_h)
-
             labels = to_one_hot(train_gt, k)
-            labels = repeat(train_gt, "n -> n v k", v=v, k=k)
-            reporter.platt_scale(labels, first_train_h)
+            labels = repeat(labels, "n k -> n v k", v=v)
 
         elif isinstance(self.net, EigenFitterConfig):
             fitter = EigenFitter(

From 89f8987cd2988b4b27483aa818c98553504e21c6 Mon Sep 17 00:00:00 2001
From: Walter Laurito <Lauritowal@yahoo.com>
Date: Wed, 23 Aug 2023 23:13:33 +0200
Subject: [PATCH 54/77] readd platt scaling

---
 elk/training/train.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/elk/training/train.py b/elk/training/train.py
index 84e81ccc..c654ca3a 100644
--- a/elk/training/train.py
+++ b/elk/training/train.py
@@ -84,6 +84,7 @@ def apply_to_layer(
             train_loss = reporter.fit(first_train_h)
             labels = to_one_hot(train_gt, k)
             labels = repeat(labels, "n k -> n v k", v=v)
+            reporter.platt_scale(labels, first_train_h)
 
         elif isinstance(self.net, EigenFitterConfig):
             fitter = EigenFitter(

From f678c90220de3d47ee5dc8dac97c65ac4f899cb9 Mon Sep 17 00:00:00 2001
From: Walter Laurito <Lauritowal@yahoo.com>
Date: Thu, 24 Aug 2023 00:13:18 +0200
Subject: [PATCH 55/77] remove platt scaling paramters from param

---
 elk/training/ccs_reporter.py | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/elk/training/ccs_reporter.py b/elk/training/ccs_reporter.py
index 1c65d98d..7a55a858 100644
--- a/elk/training/ccs_reporter.py
+++ b/elk/training/ccs_reporter.py
@@ -9,6 +9,7 @@
 import torch.nn as nn
 from concept_erasure import LeaceFitter
 from torch import Tensor
+from typing_extensions import override
 
 from ..parsing import parse_loss
 from ..utils.typing import assert_type
@@ -88,7 +89,6 @@ def __init__(
         num_variants: int = 1,
     ):
         super().__init__()
-        self._is_training = True
 
         self.config = cfg
         self.in_features = in_features
@@ -130,6 +130,15 @@ def __init__(
                 )
             )
 
+    @override
+    def parameters(self, recurse=True):
+        parameters = super(CcsReporter, self).parameters(recurse=recurse)
+        for param in parameters:
+            # exclude the platt scaling parameters
+            # kind of a hack for now, we should find probably a cleaner way
+            if param is not self.scale and param is not self.bias:
+                yield param
+
     def reset_parameters(self):
         """Reset the parameters of the probe.
 
@@ -167,12 +176,8 @@ def forward(self, x: Tensor) -> Tensor:
         """Return the credence assigned to the hidden state `x`."""
         assert self.norm is not None, "Must call fit() before forward()"
         raw_scores = self.probe(self.norm(x)).squeeze(-1)
-        if self._is_training:
-            return raw_scores
-        else:
-            # only do platt scaling after training the reporters
-            platt_scaled_scores = raw_scores.mul(self.scale).add(self.bias).squeeze(-1)
-            return platt_scaled_scores
+        platt_scaled_scores = raw_scores.mul(self.scale).add(self.bias).squeeze(-1)
+        return platt_scaled_scores
 
     def loss(self, logit0: Tensor, logit1: Tensor) -> Tensor:
         """Return the loss of the reporter on the contrast pair (x0, x1).
@@ -249,7 +254,6 @@ def fit(self, hiddens: Tensor) -> float:
 
         self.load_state_dict(best_state)
 
-        self._is_training = False
         return best_loss
 
     def train_loop_adam(self, x_neg: Tensor, x_pos: Tensor) -> float:

From 6ed5eca7eb35967e5117f83179d21ba1e7ba145d Mon Sep 17 00:00:00 2001
From: Walter Laurito <Lauritowal@yahoo.com>
Date: Mon, 28 Aug 2023 00:02:28 +0200
Subject: [PATCH 56/77] remove extra line for label

---
 elk/training/train.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/elk/training/train.py b/elk/training/train.py
index c654ca3a..fb882240 100644
--- a/elk/training/train.py
+++ b/elk/training/train.py
@@ -82,8 +82,7 @@ def apply_to_layer(
 
             reporter = CcsReporter(self.net, d, device=device, num_variants=v)
             train_loss = reporter.fit(first_train_h)
-            labels = to_one_hot(train_gt, k)
-            labels = repeat(labels, "n k -> n v k", v=v)
+            labels = repeat(to_one_hot(train_gt, k), "n k -> n v k", v=v)
             reporter.platt_scale(labels, first_train_h)
 
         elif isinstance(self.net, EigenFitterConfig):

From 728cd42b79f4cd19e4642e6243c8e33c417da69b Mon Sep 17 00:00:00 2001
From: jon <jonathan.n@u.nus.edu>
Date: Thu, 31 Aug 2023 23:28:32 +0300
Subject: [PATCH 57/77] add none in acc

black'd
---
 elk/metrics/eval.py | 32 ++++++++++++++++++++++++++------
 1 file changed, 26 insertions(+), 6 deletions(-)

diff --git a/elk/metrics/eval.py b/elk/metrics/eval.py
index 653beae5..d0e2bf7a 100644
--- a/elk/metrics/eval.py
+++ b/elk/metrics/eval.py
@@ -23,6 +23,8 @@ class EvalResult:
     roc_auc: RocAucResult
     """Area under the ROC curve. For multi-class classification, each class is treated
     as a one-vs-rest binary classification problem."""
+    cal_thresh: float | None
+    """The threshold used to compute the calibrated accuracy."""
 
     def to_dict(self, prefix: str = "") -> dict[str, float]:
         """Convert the result to a dictionary."""
@@ -38,7 +40,13 @@ def to_dict(self, prefix: str = "") -> dict[str, float]:
             else {}
         )
         auroc_dict = {f"{prefix}auroc_{k}": v for k, v in asdict(self.roc_auc).items()}
-        return {**auroc_dict, **cal_acc_dict, **acc_dict, **cal_dict}
+        return {
+            **auroc_dict,
+            **cal_acc_dict,
+            **acc_dict,
+            **cal_dict,
+            f"{prefix}cal_thresh": self.cal_thresh,
+        }
 
 
 def evaluate_preds(
@@ -64,7 +72,14 @@ def evaluate_preds(
     else:
         y_true = repeat(y_true, "n -> n v", v=v)
 
-    y_pred = y_logits.argmax(dim=-1)
+    THRESHOLD = 0.5
+    if ensembling == "none":
+        y_pred = y_logits[..., 1].gt(THRESHOLD).to(torch.int)
+    else:
+        y_pred = y_logits.argmax(dim=-1)
+
+    acc = accuracy_ci(y_true, y_pred)
+
     if ensembling == "none":
         auroc = roc_auc_ci(to_one_hot(y_true, c).long().flatten(1), y_logits.flatten(1))
     elif ensembling in ("partial", "full"):
@@ -76,22 +91,27 @@ def evaluate_preds(
     else:
         raise ValueError(f"Unknown mode: {ensembling}")
 
-    acc = accuracy_ci(y_true, y_pred)
     cal_acc = None
     cal_err = None
+    cal_thresh = None
 
     if c == 2:
-        pos_probs = torch.sigmoid(y_logits[..., 1] - y_logits[..., 0])
+        pooled_logits = (
+            y_logits[..., 1]
+            if ensembling == "none"
+            else y_logits[..., 1] - y_logits[..., 0]
+        )
+        pos_probs = torch.sigmoid(pooled_logits)
 
         # Calibrated accuracy
-        cal_thresh = pos_probs.float().quantile(y_true.float().mean())
+        cal_thresh = pos_probs.float().quantile(y_true.float().mean()).item()
         cal_preds = pos_probs.gt(cal_thresh).to(torch.int)
         cal_acc = accuracy_ci(y_true, cal_preds)
 
         cal = CalibrationError().update(y_true.flatten(), pos_probs.flatten())
         cal_err = cal.compute()
 
-    return EvalResult(acc, cal_acc, cal_err, auroc)
+    return EvalResult(acc, cal_acc, cal_err, auroc, cal_thresh)
 
 
 def to_one_hot(labels: Tensor, n_classes: int) -> Tensor:

From 1476469f61ded356e2bc694c7ad9f718de3a9ee9 Mon Sep 17 00:00:00 2001
From: Nora Belrose <belrose.nora@gmail.com>
Date: Thu, 31 Aug 2023 21:51:23 +0000
Subject: [PATCH 58/77] Add log softmax to model_logits

---
 elk/extraction/extraction.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/elk/extraction/extraction.py b/elk/extraction/extraction.py
index fe4fdc84..acd87100 100644
--- a/elk/extraction/extraction.py
+++ b/elk/extraction/extraction.py
@@ -329,7 +329,7 @@ def extract_hiddens(
             **hidden_dict,
         )
         if has_lm_preds:
-            out_record["model_logits"] = lm_logits
+            out_record["model_logits"] = lm_logits.log_softmax(dim=-1)
 
         num_yielded += 1
         yield out_record

From c9a8e1928444052ef267225071f62557e725765b Mon Sep 17 00:00:00 2001
From: Alex Mallen <atmallen8@gmail.com>
Date: Thu, 9 Nov 2023 19:40:37 +0000
Subject: [PATCH 59/77] fix sweep default

---
 elk/training/sweep.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/elk/training/sweep.py b/elk/training/sweep.py
index a4e5c97a..1902e0d7 100755
--- a/elk/training/sweep.py
+++ b/elk/training/sweep.py
@@ -1,4 +1,4 @@
-from dataclasses import InitVar, dataclass, replace
+from dataclasses import InitVar, dataclass, replace, field
 
 import numpy as np
 import torch
@@ -53,10 +53,11 @@ class Sweep:
     name: str | None = None
 
     # A bit of a hack to add all the command line arguments from Elicit
-    run_template: Elicit = Elicit(
-        data=Extract(
-            model="<placeholder>",
-            datasets=("<placeholder>",),
+    run_template: Elicit = field(default_factory=lambda: Elicit(
+            data=Extract(
+                model="<placeholder>",
+                datasets=("<placeholder>",),
+            )
         )
     )
 

From 611c93ca02294bd9593104c2db78bffe266f5204 Mon Sep 17 00:00:00 2001
From: Alex Mallen <atmallen8@gmail.com>
Date: Fri, 10 Nov 2023 17:29:59 +0000
Subject: [PATCH 60/77] Merge with EleutherELK

---
 elk/training/sweep.py | 11 ++---------
 elk/training/train.py | 10 ++++++++++
 pyproject.toml        |  2 ++
 3 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/elk/training/sweep.py b/elk/training/sweep.py
index 1902e0d7..c9ca7fc9 100755
--- a/elk/training/sweep.py
+++ b/elk/training/sweep.py
@@ -1,4 +1,4 @@
-from dataclasses import InitVar, dataclass, replace, field
+from dataclasses import InitVar, dataclass, field, replace
 
 import numpy as np
 import torch
@@ -6,7 +6,6 @@
 from transformers import AutoConfig
 
 from ..evaluation import Eval
-from ..extraction import Extract
 from ..files import memorably_named_dir, sweeps_dir
 from ..plotting.visualize import visualize_sweep
 from ..training.eigen_reporter import EigenFitterConfig
@@ -53,13 +52,7 @@ class Sweep:
     name: str | None = None
 
     # A bit of a hack to add all the command line arguments from Elicit
-    run_template: Elicit = field(default_factory=lambda: Elicit(
-            data=Extract(
-                model="<placeholder>",
-                datasets=("<placeholder>",),
-            )
-        )
-    )
+    run_template: Elicit = field(default_factory=Elicit.default)
 
     def __post_init__(self, add_pooled: bool):
         if not self.datasets:
diff --git a/elk/training/train.py b/elk/training/train.py
index fb882240..3d00b54c 100644
--- a/elk/training/train.py
+++ b/elk/training/train.py
@@ -11,6 +11,7 @@
 from simple_parsing import subgroups
 from simple_parsing.helpers.serialization import save
 
+from ..extraction import Extract
 from ..metrics import evaluate_preds, to_one_hot
 from ..run import Run
 from ..training.supervised import train_supervised
@@ -34,6 +35,15 @@ class Elicit(Run):
     cross-validation. Defaults to "single", which means to train a single classifier
     on the training data. "cv" means to use cross-validation."""
 
+    @staticmethod
+    def default():
+        return Elicit(
+            data=Extract(
+                model="<placeholder>",
+                datasets=("<placeholder>",),
+            )
+        )
+
     def create_models_dir(self, out_dir: Path):
         lr_dir = None
         lr_dir = out_dir / "lr_models"
diff --git a/pyproject.toml b/pyproject.toml
index f3f16504..cb004269 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -12,6 +12,8 @@ license = { text = "MIT License" }
 dependencies = [
     # Allows us to use device_map in from_pretrained. Also needed for 8bit
     "accelerate",
+    # Already a dependency of datasets, but newer versions introduce breaking changes
+    "pyarrow==12.0.0",
     # For pseudolabel and prompt normalization. We're picky about the version because
     # the package isn't guaranteed to be stable yet.
     "concept-erasure==0.1.0",

From 71d79a279045d9405a5333545dccdfe207ad6fbc Mon Sep 17 00:00:00 2001
From: Alex Mallen <atmallen8@gmail.com>
Date: Fri, 10 Nov 2023 18:15:10 +0000
Subject: [PATCH 61/77] rename module from elk to ccs

---
 .gitignore                                    |  4 ---
 MANIFEST.in                                   |  4 +--
 README.md                                     | 24 +++++++-------
 {elk => ccs}/__init__.py                      |  0
 {elk => ccs}/__main__.py                      | 10 +++---
 {elk => ccs}/debug_logging.py                 |  0
 {elk => ccs}/evaluation/__init__.py           |  0
 {elk => ccs}/evaluation/evaluate.py           |  6 ++--
 {elk => ccs}/extraction/__init__.py           |  0
 {elk => ccs}/extraction/balanced_sampler.py   |  0
 {elk => ccs}/extraction/dataset_name.py       |  0
 {elk => ccs}/extraction/extraction.py         |  0
 {elk => ccs}/extraction/generator.py          |  0
 {elk => ccs}/extraction/prompt_loading.py     |  0
 {elk => ccs}/files.py                         | 10 +++---
 {elk => ccs}/metrics/__init__.py              |  0
 {elk => ccs}/metrics/accuracy.py              |  0
 {elk => ccs}/metrics/calibration.py           |  0
 {elk => ccs}/metrics/eval.py                  |  0
 {elk => ccs}/metrics/roc_auc.py               |  0
 {elk => ccs}/parsing.py                       |  0
 {elk => ccs}/plotting/command.py              |  0
 {elk => ccs}/plotting/visualize.py            |  0
 {elk => ccs}/promptsource/LICENSE             |  0
 {elk => ccs}/promptsource/__init__.py         |  0
 {elk => ccs}/promptsource/templates.py        |  0
 .../Zaid/coqa_expanded/templates.yaml         |  0
 .../Zaid/quac_expanded/templates.yaml         |  0
 .../acronym_identification/templates.yaml     |  0
 .../templates.yaml                            |  0
 .../templates.yaml                            |  0
 .../templates.yaml                            |  0
 .../adversarialQA/templates.yaml              |  0
 .../adversarial_qa/dbert/templates.yaml       |  0
 .../adversarial_qa/dbidaf/templates.yaml      |  0
 .../adversarial_qa/droberta/templates.yaml    |  0
 .../templates/aeslc/templates.yaml            |  0
 .../templates/ag_news/templates.yaml          |  0
 .../ai2_arc/ARC-Challenge/templates.yaml      |  0
 .../templates/ai2_arc/ARC-Easy/templates.yaml |  0
 .../templates/amazon_polarity/templates.yaml  |  0
 .../amazon_reviews_multi/en/templates.yaml    |  0
 .../Wireless_v1_00/templates.yaml             |  0
 .../templates/ambig_qa/light/templates.yaml   |  0
 .../templates/anli/templates.yaml             |  0
 .../templates/app_reviews/templates.yaml      |  0
 .../templates/aqua_rat/raw/templates.yaml     |  0
 .../promptsource/templates/art/templates.yaml |  0
 .../templates/asnq/templates.yaml             |  0
 .../templates/asset/ratings/templates.yaml    |  0
 .../asset/simplification/templates.yaml       |  0
 .../templates/banking77/templates.yaml        |  0
 .../templates/billsum/templates.yaml          |  0
 .../bing_coronavirus_query_set/templates.yaml |  0
 .../templates/biosses/templates.yaml          |  0
 .../title_genre_classifiction/templates.yaml  |  0
 .../blended_skill_talk/templates.yaml         |  0
 .../templates/boolq_pt/templates.yaml         |  0
 .../templates/cbt/CN/templates.yaml           |  0
 .../templates/cbt/NE/templates.yaml           |  0
 .../templates/cbt/P/templates.yaml            |  0
 .../templates/cbt/V/templates.yaml            |  0
 .../templates/cbt/raw/templates.yaml          |  0
 .../templates/cc_news/templates.yaml          |  0
 .../christykoh/ag_news_pt/templates.yaml      |  0
 .../christykoh/boolq_pt/templates.yaml        |  0
 .../christykoh/imdb_ar/templates.yaml         |  0
 .../christykoh/imdb_es/templates.yaml         |  0
 .../christykoh/imdb_fr/templates.yaml         |  0
 .../christykoh/imdb_pt/templates.yaml         |  0
 .../christykoh/imdb_zh/templates.yaml         |  0
 .../templates/circa/templates.yaml            |  0
 .../templates/climate_fever/templates.yaml    |  0
 .../cnn_dailymail/3.0.0/templates.yaml        |  0
 .../templates/codah/codah/templates.yaml      |  0
 .../templates/codah/fold_0/templates.yaml     |  0
 .../templates/codah/fold_1/templates.yaml     |  0
 .../templates/codah/fold_2/templates.yaml     |  0
 .../templates/codah/fold_3/templates.yaml     |  0
 .../templates/codah/fold_4/templates.yaml     |  0
 .../templates.yaml                            |  0
 .../templates/common_gen/templates.yaml       |  0
 .../templates/commonsense_qa/templates.yaml   |  0
 .../templates/conv_ai/templates.yaml          |  0
 .../templates/conv_ai_2/templates.yaml        |  0
 .../templates/conv_ai_3/templates.yaml        |  0
 .../templates/coqa/templates.yaml             |  0
 .../templates/cord19/metadata/templates.yaml  |  0
 .../templates/cos_e/v1.0/templates.yaml       |  0
 .../templates/cos_e/v1.11/templates.yaml      |  0
 .../templates/cosmos_qa/templates.yaml        |  0
 .../covid_qa_castorini/templates.yaml         |  0
 .../craffel/openai_lambada/templates.yaml     |  0
 .../craigslist_bargains/templates.yaml        |  0
 .../templates/crows_pairs/templates.yaml      |  0
 .../templates/dbpedia_14/templates.yaml       |  0
 .../discofuse/discofuse-sport/templates.yaml  |  0
 .../discofuse-wikipedia/templates.yaml        |  0
 .../discovery/discovery/templates.yaml        |  0
 .../templates/docred/templates.yaml           |  0
 .../templates/dream/templates.yaml            |  0
 .../templates/drop/templates.yaml             |  0
 .../duorc/ParaphraseRC/templates.yaml         |  0
 .../templates/duorc/SelfRC/templates.yaml     |  0
 .../templates/e2e_nlg_cleaned/templates.yaml  |  0
 .../templates.yaml                            |  0
 .../promptsource/templates/emo/templates.yaml |  0
 .../templates/emotion/templates.yaml          |  0
 .../enriched_web_nlg/en/templates.yaml        |  0
 .../templates/esnli/templates.yaml            |  0
 .../1.1/templates.yaml                        |  0
 .../2.0/templates.yaml                        |  0
 .../templates/fever/v1.0/templates.yaml       |  0
 .../templates/fever/v2.0/templates.yaml       |  0
 .../sentences_allagree/templates.yaml         |  0
 .../templates/freebase_qa/templates.yaml      |  0
 .../generated_reviews_enth/templates.yaml     |  0
 .../templates/gigaword/templates.yaml         |  0
 .../templates/glue/ax/templates.yaml          |  0
 .../templates/glue/cola/templates.yaml        |  0
 .../templates/glue/mnli/templates.yaml        |  0
 .../glue/mnli_matched/templates.yaml          |  0
 .../glue/mnli_mismatched/templates.yaml       |  0
 .../templates/glue/mrpc/templates.yaml        |  0
 .../templates/glue/qnli/templates.yaml        |  0
 .../templates/glue/qqp/templates.yaml         |  0
 .../templates/glue/rte/templates.yaml         |  0
 .../templates/glue/sst2/templates.yaml        |  0
 .../templates/glue/stsb/templates.yaml        |  0
 .../templates/glue/wnli/templates.yaml        |  0
 .../google_wellformed_query/templates.yaml    |  0
 .../templates/great_code/templates.yaml       |  0
 .../cross_genre_1/templates.yaml              |  0
 .../cross_topic_1/templates.yaml              |  0
 .../cross_topic_4/templates.yaml              |  0
 .../cross_topic_7/templates.yaml              |  0
 .../templates/gutenberg_time/templates.yaml   |  0
 .../templates/hans/templates.yaml             |  0
 .../templates/hate_speech18/templates.yaml    |  0
 .../templates/head_qa/en/templates.yaml       |  0
 .../templates/health_fact/templates.yaml      |  0
 .../templates/hellaswag/templates.yaml        |  0
 .../templates/hlgd/templates.yaml             |  0
 .../hotpot_qa/distractor/templates.yaml       |  0
 .../hotpot_qa/fullwiki/templates.yaml         |  0
 .../humicroedit/subtask-1/templates.yaml      |  0
 .../humicroedit/subtask-2/templates.yaml      |  0
 .../byarticle/templates.yaml                  |  0
 .../bypublisher/templates.yaml                |  0
 .../templates/imdb/templates.yaml             |  0
 .../templates/jfleg/templates.yaml            |  0
 .../jigsaw_unintended_bias/templates.yaml     |  0
 .../templates/kelm/templates.yaml             |  0
 .../kilt_tasks/hotpotqa/templates.yaml        |  0
 .../templates/kilt_tasks/nq/templates.yaml    |  0
 .../templates/lama/trex/templates.yaml        |  0
 .../templates/lambada/templates.yaml          |  0
 .../lauritowal/redefine_math/templates.yaml   |  0
 .../templates/liar/templates.yaml             |  0
 .../templates/limit/templates.yaml            |  0
 .../algebra__linear_1d/templates.yaml         |  0
 .../templates.yaml                            |  0
 .../algebra__linear_2d/templates.yaml         |  0
 .../templates.yaml                            |  0
 .../templates/math_qa/templates.yaml          |  0
 .../templates/mc_taco/templates.yaml          |  0
 .../templates/mdd/task1_qa/templates.yaml     |  0
 .../templates/mdd/task2_recs/templates.yaml   |  0
 .../templates/mdd/task3_qarecs/templates.yaml |  0
 .../templates/medal/templates.yaml            |  0
 .../medical_questions_pairs/templates.yaml    |  0
 .../meta_woz/dialogues/templates.yaml         |  0
 .../templates/mocha/templates.yaml            |  0
 .../templates/movie_rationales/templates.yaml |  0
 .../templates/multi_news/templates.yaml       |  0
 .../templates/multi_nli/templates.yaml        |  0
 .../multi_x_science_sum/templates.yaml        |  0
 .../templates/mwsc/templates.yaml             |  0
 .../templates/narrativeqa/templates.yaml      |  0
 .../templates/ncbi_disease/templates.yaml     |  0
 .../evaluation_dataset/templates.yaml         |  0
 .../templates/newspop/templates.yaml          |  0
 .../nlu_evaluation_data/templates.yaml        |  0
 .../templates/nq_open/templates.yaml          |  0
 .../templates/numer_sense/templates.yaml      |  0
 .../templates/onestop_english/templates.yaml  |  0
 .../templates/openai_humaneval/templates.yaml |  0
 .../openbookqa/additional/templates.yaml      |  0
 .../templates/openbookqa/main/templates.yaml  |  0
 .../templates/paws-x/en/templates.yaml        |  0
 .../paws/labeled_final/templates.yaml         |  0
 .../paws/labeled_swap/templates.yaml          |  0
 .../paws/unlabeled_final/templates.yaml       |  0
 .../templates/piqa/templates.yaml             |  0
 .../templates/poem_sentiment/templates.yaml   |  0
 .../pubmed_qa/pqa_labeled/templates.yaml      |  0
 .../templates/qa_srl/templates.yaml           |  0
 .../templates/qa_zre/templates.yaml           |  0
 .../templates/qasc/templates.yaml             |  0
 .../promptsource/templates/qed/templates.yaml |  0
 .../templates/quac/templates.yaml             |  0
 .../templates/quail/templates.yaml            |  0
 .../templates/quarel/templates.yaml           |  0
 .../templates/quartz/templates.yaml           |  0
 .../templates/quora/templates.yaml            |  0
 .../templates/quoref/templates.yaml           |  0
 .../templates/race/all/templates.yaml         |  0
 .../templates/race/high/templates.yaml        |  0
 .../templates/race/middle/templates.yaml      |  0
 .../reaganjlee/boolq_ar/templates.yaml        |  0
 .../reaganjlee/boolq_es/templates.yaml        |  0
 .../reaganjlee/boolq_fr/templates.yaml        |  0
 .../reaganjlee/boolq_pt/templates.yaml        |  0
 .../reaganjlee/boolq_zh/templates.yaml        |  0
 .../reaganjlee/truthful_qa_mc/templates.yaml  |  0
 .../truthful_qa_mc_ar/templates.yaml          |  0
 .../truthful_qa_mc_es/templates.yaml          |  0
 .../truthful_qa_mc_fr/templates.yaml          |  0
 .../truthful_qa_mc_pt/templates.yaml          |  0
 .../truthful_qa_mc_zh/templates.yaml          |  0
 .../templates/riddle_sense/templates.yaml     |  0
 .../templates/ropes/templates.yaml            |  0
 .../templates/rotten_tomatoes/templates.yaml  |  0
 .../templates/samsum/templates.yaml           |  0
 .../scan/addprim_jump/templates.yaml          |  0
 .../scan/addprim_turn_left/templates.yaml     |  0
 .../templates/scan/filler_num0/templates.yaml |  0
 .../templates/scan/filler_num1/templates.yaml |  0
 .../templates/scan/filler_num2/templates.yaml |  0
 .../templates/scan/filler_num3/templates.yaml |  0
 .../templates/scan/length/templates.yaml      |  0
 .../templates/scan/simple/templates.yaml      |  0
 .../scan/template_around_right/templates.yaml |  0
 .../template_jump_around_right/templates.yaml |  0
 .../template_opposite_right/templates.yaml    |  0
 .../scan/template_right/templates.yaml        |  0
 .../templates/scicite/templates.yaml          |  0
 .../scientific_papers/arxiv/templates.yaml    |  0
 .../scientific_papers/pubmed/templates.yaml   |  0
 .../templates/sciq/templates.yaml             |  0
 .../scitail/snli_format/templates.yaml        |  0
 .../scitail/tsv_format/templates.yaml         |  0
 .../templates/scitldr/Abstract/templates.yaml |  0
 .../answer_selection_analysis/templates.yaml  |  0
 .../sem_eval_2010_task_8/templates.yaml       |  0
 .../sem_eval_2014_task_1/templates.yaml       |  0
 .../templates/sent_comp/templates.yaml        |  0
 .../templates/sick/templates.yaml             |  0
 .../templates/sms_spam/templates.yaml         |  0
 .../snips_built_in_intents/templates.yaml     |  0
 .../templates/snli/templates.yaml             |  0
 .../templates/social_i_qa/templates.yaml      |  0
 .../templates/species_800/templates.yaml      |  0
 .../templates/squad/templates.yaml            |  0
 .../squad_adversarial/AddSent/templates.yaml  |  0
 .../templates/squad_v2/templates.yaml         |  0
 .../squadshifts/amazon/templates.yaml         |  0
 .../squadshifts/new_wiki/templates.yaml       |  0
 .../templates/squadshifts/nyt/templates.yaml  |  0
 .../templates/sst/default/templates.yaml      |  0
 .../templates/sst2/templates.yaml             |  0
 .../templates/story_cloze/2016/templates.yaml |  0
 .../templates/stsb_multi_mt/en/templates.yaml |  0
 .../templates/subjqa/books/templates.yaml     |  0
 .../subjqa/electronics/templates.yaml         |  0
 .../templates/subjqa/grocery/templates.yaml   |  0
 .../templates/subjqa/movies/templates.yaml    |  0
 .../subjqa/restaurants/templates.yaml         |  0
 .../subjqa/tripadvisor/templates.yaml         |  0
 .../templates/super_glue/axb/templates.yaml   |  0
 .../templates/super_glue/axg/templates.yaml   |  0
 .../templates/super_glue/boolq/templates.yaml |  0
 .../templates/super_glue/cb/templates.yaml    |  0
 .../templates/super_glue/copa/templates.yaml  |  0
 .../super_glue/multirc/templates.yaml         |  0
 .../super_glue/record/templates.yaml          |  0
 .../templates/super_glue/rte/templates.yaml   |  0
 .../templates/super_glue/wic/templates.yaml   |  0
 .../super_glue/wsc.fixed/templates.yaml       |  0
 .../templates/swag/regular/templates.yaml     |  0
 .../tab_fact/tab_fact/templates.yaml          |  0
 .../templates/tmu_gfm_dataset/templates.yaml  |  0
 .../templates/trec/templates.yaml             |  0
 .../trivia_qa/unfiltered/templates.yaml       |  0
 .../templates/turk/templates.yaml             |  0
 .../templates/tweet_eval/emoji/templates.yaml |  0
 .../tweet_eval/emotion/templates.yaml         |  0
 .../templates/tweet_eval/hate/templates.yaml  |  0
 .../templates/tweet_eval/irony/templates.yaml |  0
 .../tweet_eval/offensive/templates.yaml       |  0
 .../tweet_eval/sentiment/templates.yaml       |  0
 .../tweet_eval/stance_abortion/templates.yaml |  0
 .../tweet_eval/stance_atheism/templates.yaml  |  0
 .../tweet_eval/stance_climate/templates.yaml  |  0
 .../tweet_eval/stance_feminist/templates.yaml |  0
 .../tweet_eval/stance_hillary/templates.yaml  |  0
 .../tydiqa/primary_task/templates.yaml        |  0
 .../tydiqa/secondary_task/templates.yaml      |  0
 .../templates/web_questions/templates.yaml    |  0
 .../templates/wiki_bio/templates.yaml         |  0
 .../templates/wiki_hop/masked/templates.yaml  |  0
 .../wiki_hop/original/templates.yaml          |  0
 .../templates/wiki_qa/templates.yaml          |  0
 .../templates/wiki_split/templates.yaml       |  0
 .../wino_bias/type1_anti/templates.yaml       |  0
 .../wino_bias/type1_pro/templates.yaml        |  0
 .../wino_bias/type2_anti/templates.yaml       |  0
 .../wino_bias/type2_pro/templates.yaml        |  0
 .../winograd_wsc/wsc273/templates.yaml        |  0
 .../winograd_wsc/wsc285/templates.yaml        |  0
 .../winogrande_debiased/templates.yaml        |  0
 .../winogrande/winogrande_l/templates.yaml    |  0
 .../winogrande/winogrande_m/templates.yaml    |  0
 .../winogrande/winogrande_s/templates.yaml    |  0
 .../winogrande/winogrande_xl/templates.yaml   |  0
 .../winogrande/winogrande_xs/templates.yaml   |  0
 .../templates/wiqa/templates.yaml             |  0
 .../templates/xnli/en/templates.yaml          |  0
 .../templates/xquad/xquad.en/templates.yaml   |  0
 .../templates/xquad_r/en/templates.yaml       |  0
 .../templates/xsum/templates.yaml             |  0
 .../templates/yahoo_answers_qa/templates.yaml |  0
 .../yahoo_answers_topics/templates.yaml       |  0
 .../templates/yelp_polarity/templates.yaml    |  0
 .../templates/yelp_review_full/templates.yaml |  0
 .../templates/zest/templates.yaml             |  0
 {elk => ccs}/resources/adjectives.json        |  0
 {elk => ccs}/resources/names.json             |  0
 {elk => ccs}/run.py                           |  6 ++--
 {elk => ccs}/training/__init__.py             |  0
 {elk => ccs}/training/burns_norm.py           |  0
 {elk => ccs}/training/ccs_reporter.py         |  2 +-
 {elk => ccs}/training/classifier.py           |  0
 {elk => ccs}/training/common.py               |  0
 {elk => ccs}/training/eigen_reporter.py       |  0
 {elk => ccs}/training/losses.py               |  0
 {elk => ccs}/training/platt_scaling.py        |  0
 {elk => ccs}/training/supervised.py           |  0
 {elk => ccs}/training/sweep.py                |  0
 {elk => ccs}/training/train.py                |  0
 {elk => ccs}/truncated_eigh.py                |  0
 {elk => ccs}/utils/__init__.py                |  0
 {elk => ccs}/utils/constants.py               |  0
 {elk => ccs}/utils/data_utils.py              |  0
 {elk => ccs}/utils/gpu_utils.py               |  0
 {elk => ccs}/utils/hf_utils.py                |  0
 {elk => ccs}/utils/math_util.py               |  0
 {elk => ccs}/utils/pretty.py                  |  0
 {elk => ccs}/utils/tree_utils.py              |  0
 {elk => ccs}/utils/typing.py                  |  0
 pyproject.toml                                | 12 +++----
 tests/test_burns_norm.py                      |  2 +-
 tests/test_classifier.py                      |  2 +-
 tests/test_eigen_reporter.py                  |  4 +--
 tests/test_load_prompts.py                    |  4 +--
 tests/test_math.py                            |  2 +-
 tests/test_metrics.py                         |  2 +-
 tests/test_samplers.py                        |  4 +--
 tests/test_smoke_elicit.py                    |  6 ++--
 tests/test_smoke_eval.py                      |  8 ++---
 tests/test_truncated_eigh.py                  |  2 +-
 tests/test_viz.py                             | 31 -------------------
 362 files changed, 55 insertions(+), 90 deletions(-)
 rename {elk => ccs}/__init__.py (100%)
 rename {elk => ccs}/__main__.py (71%)
 rename {elk => ccs}/debug_logging.py (100%)
 rename {elk => ccs}/evaluation/__init__.py (100%)
 rename {elk => ccs}/evaluation/evaluate.py (95%)
 rename {elk => ccs}/extraction/__init__.py (100%)
 rename {elk => ccs}/extraction/balanced_sampler.py (100%)
 rename {elk => ccs}/extraction/dataset_name.py (100%)
 rename {elk => ccs}/extraction/extraction.py (100%)
 rename {elk => ccs}/extraction/generator.py (100%)
 rename {elk => ccs}/extraction/prompt_loading.py (100%)
 rename {elk => ccs}/files.py (84%)
 rename {elk => ccs}/metrics/__init__.py (100%)
 rename {elk => ccs}/metrics/accuracy.py (100%)
 rename {elk => ccs}/metrics/calibration.py (100%)
 rename {elk => ccs}/metrics/eval.py (100%)
 rename {elk => ccs}/metrics/roc_auc.py (100%)
 rename {elk => ccs}/parsing.py (100%)
 rename {elk => ccs}/plotting/command.py (100%)
 rename {elk => ccs}/plotting/visualize.py (100%)
 rename {elk => ccs}/promptsource/LICENSE (100%)
 rename {elk => ccs}/promptsource/__init__.py (100%)
 rename {elk => ccs}/promptsource/templates.py (100%)
 rename {elk => ccs}/promptsource/templates/Zaid/coqa_expanded/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/Zaid/quac_expanded/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/acronym_identification/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/ade_corpus_v2/Ade_corpus_v2_classification/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/ade_corpus_v2/Ade_corpus_v2_drug_ade_relation/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/ade_corpus_v2/Ade_corpus_v2_drug_dosage_relation/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/adversarial_qa/adversarialQA/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/adversarial_qa/dbert/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/adversarial_qa/dbidaf/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/adversarial_qa/droberta/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/aeslc/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/ag_news/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/ai2_arc/ARC-Challenge/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/ai2_arc/ARC-Easy/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/amazon_polarity/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/amazon_reviews_multi/en/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/amazon_us_reviews/Wireless_v1_00/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/ambig_qa/light/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/anli/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/app_reviews/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/aqua_rat/raw/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/art/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/asnq/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/asset/ratings/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/asset/simplification/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/banking77/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/billsum/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/bing_coronavirus_query_set/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/biosses/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/blbooksgenre/title_genre_classifiction/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/blended_skill_talk/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/boolq_pt/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/cbt/CN/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/cbt/NE/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/cbt/P/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/cbt/V/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/cbt/raw/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/cc_news/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/christykoh/ag_news_pt/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/christykoh/boolq_pt/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/christykoh/imdb_ar/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/christykoh/imdb_es/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/christykoh/imdb_fr/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/christykoh/imdb_pt/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/christykoh/imdb_zh/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/circa/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/climate_fever/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/cnn_dailymail/3.0.0/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/codah/codah/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/codah/fold_0/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/codah/fold_1/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/codah/fold_2/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/codah/fold_3/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/codah/fold_4/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/code_x_glue_tc_text_to_code/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/common_gen/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/commonsense_qa/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/conv_ai/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/conv_ai_2/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/conv_ai_3/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/coqa/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/cord19/metadata/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/cos_e/v1.0/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/cos_e/v1.11/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/cosmos_qa/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/covid_qa_castorini/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/craffel/openai_lambada/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/craigslist_bargains/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/crows_pairs/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/dbpedia_14/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/discofuse/discofuse-sport/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/discofuse/discofuse-wikipedia/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/discovery/discovery/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/docred/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/dream/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/drop/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/duorc/ParaphraseRC/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/duorc/SelfRC/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/e2e_nlg_cleaned/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/ecthr_cases/alleged-violation-prediction/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/emo/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/emotion/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/enriched_web_nlg/en/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/esnli/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/evidence_infer_treatment/1.1/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/evidence_infer_treatment/2.0/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/fever/v1.0/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/fever/v2.0/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/financial_phrasebank/sentences_allagree/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/freebase_qa/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/generated_reviews_enth/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/gigaword/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/glue/ax/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/glue/cola/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/glue/mnli/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/glue/mnli_matched/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/glue/mnli_mismatched/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/glue/mrpc/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/glue/qnli/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/glue/qqp/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/glue/rte/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/glue/sst2/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/glue/stsb/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/glue/wnli/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/google_wellformed_query/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/great_code/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/guardian_authorship/cross_genre_1/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/guardian_authorship/cross_topic_1/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/guardian_authorship/cross_topic_4/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/guardian_authorship/cross_topic_7/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/gutenberg_time/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/hans/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/hate_speech18/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/head_qa/en/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/health_fact/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/hellaswag/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/hlgd/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/hotpot_qa/distractor/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/hotpot_qa/fullwiki/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/humicroedit/subtask-1/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/humicroedit/subtask-2/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/hyperpartisan_news_detection/byarticle/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/hyperpartisan_news_detection/bypublisher/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/imdb/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/jfleg/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/jigsaw_unintended_bias/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/kelm/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/kilt_tasks/hotpotqa/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/kilt_tasks/nq/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/lama/trex/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/lambada/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/lauritowal/redefine_math/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/liar/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/limit/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/math_dataset/algebra__linear_1d/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/math_dataset/algebra__linear_1d_composed/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/math_dataset/algebra__linear_2d/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/math_dataset/algebra__linear_2d_composed/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/math_qa/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/mc_taco/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/mdd/task1_qa/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/mdd/task2_recs/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/mdd/task3_qarecs/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/medal/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/medical_questions_pairs/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/meta_woz/dialogues/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/mocha/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/movie_rationales/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/multi_news/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/multi_nli/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/multi_x_science_sum/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/mwsc/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/narrativeqa/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/ncbi_disease/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/neural_code_search/evaluation_dataset/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/newspop/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/nlu_evaluation_data/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/nq_open/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/numer_sense/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/onestop_english/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/openai_humaneval/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/openbookqa/additional/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/openbookqa/main/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/paws-x/en/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/paws/labeled_final/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/paws/labeled_swap/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/paws/unlabeled_final/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/piqa/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/poem_sentiment/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/pubmed_qa/pqa_labeled/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/qa_srl/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/qa_zre/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/qasc/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/qed/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/quac/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/quail/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/quarel/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/quartz/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/quora/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/quoref/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/race/all/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/race/high/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/race/middle/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/reaganjlee/boolq_ar/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/reaganjlee/boolq_es/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/reaganjlee/boolq_fr/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/reaganjlee/boolq_pt/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/reaganjlee/boolq_zh/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/reaganjlee/truthful_qa_mc/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/reaganjlee/truthful_qa_mc_ar/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/reaganjlee/truthful_qa_mc_es/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/reaganjlee/truthful_qa_mc_fr/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/reaganjlee/truthful_qa_mc_pt/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/reaganjlee/truthful_qa_mc_zh/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/riddle_sense/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/ropes/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/rotten_tomatoes/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/samsum/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/scan/addprim_jump/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/scan/addprim_turn_left/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/scan/filler_num0/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/scan/filler_num1/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/scan/filler_num2/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/scan/filler_num3/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/scan/length/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/scan/simple/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/scan/template_around_right/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/scan/template_jump_around_right/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/scan/template_opposite_right/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/scan/template_right/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/scicite/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/scientific_papers/arxiv/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/scientific_papers/pubmed/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/sciq/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/scitail/snli_format/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/scitail/tsv_format/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/scitldr/Abstract/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/selqa/answer_selection_analysis/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/sem_eval_2010_task_8/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/sem_eval_2014_task_1/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/sent_comp/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/sick/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/sms_spam/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/snips_built_in_intents/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/snli/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/social_i_qa/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/species_800/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/squad/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/squad_adversarial/AddSent/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/squad_v2/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/squadshifts/amazon/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/squadshifts/new_wiki/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/squadshifts/nyt/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/sst/default/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/sst2/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/story_cloze/2016/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/stsb_multi_mt/en/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/subjqa/books/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/subjqa/electronics/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/subjqa/grocery/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/subjqa/movies/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/subjqa/restaurants/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/subjqa/tripadvisor/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/super_glue/axb/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/super_glue/axg/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/super_glue/boolq/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/super_glue/cb/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/super_glue/copa/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/super_glue/multirc/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/super_glue/record/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/super_glue/rte/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/super_glue/wic/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/super_glue/wsc.fixed/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/swag/regular/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/tab_fact/tab_fact/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/tmu_gfm_dataset/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/trec/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/trivia_qa/unfiltered/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/turk/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/tweet_eval/emoji/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/tweet_eval/emotion/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/tweet_eval/hate/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/tweet_eval/irony/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/tweet_eval/offensive/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/tweet_eval/sentiment/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/tweet_eval/stance_abortion/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/tweet_eval/stance_atheism/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/tweet_eval/stance_climate/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/tweet_eval/stance_feminist/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/tweet_eval/stance_hillary/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/tydiqa/primary_task/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/tydiqa/secondary_task/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/web_questions/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/wiki_bio/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/wiki_hop/masked/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/wiki_hop/original/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/wiki_qa/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/wiki_split/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/wino_bias/type1_anti/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/wino_bias/type1_pro/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/wino_bias/type2_anti/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/wino_bias/type2_pro/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/winograd_wsc/wsc273/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/winograd_wsc/wsc285/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/winogrande/winogrande_debiased/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/winogrande/winogrande_l/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/winogrande/winogrande_m/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/winogrande/winogrande_s/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/winogrande/winogrande_xl/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/winogrande/winogrande_xs/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/wiqa/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/xnli/en/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/xquad/xquad.en/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/xquad_r/en/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/xsum/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/yahoo_answers_qa/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/yahoo_answers_topics/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/yelp_polarity/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/yelp_review_full/templates.yaml (100%)
 rename {elk => ccs}/promptsource/templates/zest/templates.yaml (100%)
 rename {elk => ccs}/resources/adjectives.json (100%)
 rename {elk => ccs}/resources/names.json (100%)
 rename {elk => ccs}/run.py (97%)
 rename {elk => ccs}/training/__init__.py (100%)
 rename {elk => ccs}/training/burns_norm.py (100%)
 rename {elk => ccs}/training/ccs_reporter.py (99%)
 rename {elk => ccs}/training/classifier.py (100%)
 rename {elk => ccs}/training/common.py (100%)
 rename {elk => ccs}/training/eigen_reporter.py (100%)
 rename {elk => ccs}/training/losses.py (100%)
 rename {elk => ccs}/training/platt_scaling.py (100%)
 rename {elk => ccs}/training/supervised.py (100%)
 rename {elk => ccs}/training/sweep.py (100%)
 rename {elk => ccs}/training/train.py (100%)
 rename {elk => ccs}/truncated_eigh.py (100%)
 rename {elk => ccs}/utils/__init__.py (100%)
 rename {elk => ccs}/utils/constants.py (100%)
 rename {elk => ccs}/utils/data_utils.py (100%)
 rename {elk => ccs}/utils/gpu_utils.py (100%)
 rename {elk => ccs}/utils/hf_utils.py (100%)
 rename {elk => ccs}/utils/math_util.py (100%)
 rename {elk => ccs}/utils/pretty.py (100%)
 rename {elk => ccs}/utils/tree_utils.py (100%)
 rename {elk => ccs}/utils/typing.py (100%)
 delete mode 100644 tests/test_viz.py

diff --git a/.gitignore b/.gitignore
index 6c29ee26..2704fbd8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,10 +1,6 @@
 *.csv
 *.npy
-elk/models/*
-elk/trained/*
-nohup.out
 .idea
-*.pkl
 
 # scripts for experiments in progress
 my_*.sh
diff --git a/MANIFEST.in b/MANIFEST.in
index 416520ef..d68d1589 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,2 +1,2 @@
-recursive-include elk/promptsource/templates *
-recursive-include elk/resources *
+recursive-include ccs/promptsource/templates *
+recursive-include ccs/resources *
diff --git a/README.md b/README.md
index fa74d900..10b8166a 100644
--- a/README.md
+++ b/README.md
@@ -21,16 +21,16 @@ Our code is based on [PyTorch](http://pytorch.org)
 and [Huggingface Transformers](https://huggingface.co/docs/transformers/index). We test the code on Python 3.10 and
 3.11.
 
-First install the package with `pip install -e .` in the root directory, or `pip install eleuther-elk` to install from PyPi. Use `pip install -e .[dev]` if you'd like to contribute to the project (see **Development** section below). This should install all the necessary dependencies.
+First install the package with `pip install -e .` in the root directory. Use `pip install -e .[dev]` if you'd like to contribute to the project (see **Development** section below). This should install all the necessary dependencies.
 
 To fit reporters for the HuggingFace model `model` and dataset `dataset`, just run:
 
 ```bash
-elk elicit microsoft/deberta-v2-xxlarge-mnli imdb
+ccs elicit microsoft/deberta-v2-xxlarge-mnli imdb
 ```
 
 This will automatically download the model and dataset, run the model and extract the relevant representations if they
-aren't cached on disk, fit reporters on them, and save the reporter checkpoints to the `elk-reporters` folder in your
+aren't cached on disk, fit reporters on them, and save the reporter checkpoints to the `ccs-reporters` folder in your
 home directory. It will also evaluate the reporter classification performance on a held out test set and save it to a
 CSV file in the same folder.
 
@@ -38,35 +38,35 @@ The following will generate a CCS (Contrast Consistent Search) reporter instead
 default.
 
 ```bash
-elk elicit microsoft/deberta-v2-xxlarge-mnli imdb --net ccs
+ccs elicit microsoft/deberta-v2-xxlarge-mnli imdb --net ccs
 ```
 
 The following command will evaluate the probe from the run naughty-northcutt on the hidden states extracted from the
 model deberta-v2-xxlarge-mnli for the imdb dataset. It will result in an `eval.csv` and `cfg.yaml` file, which are
-stored under a subfolder in `elk-reporters/naughty-northcutt/transfer_eval`.
+stored under a subfolder in `ccs-reporters/naughty-northcutt/transfer_eval`.
 
 ```bash
-elk eval naughty-northcutt microsoft/deberta-v2-xxlarge-mnli imdb
+ccs eval naughty-northcutt microsoft/deberta-v2-xxlarge-mnli imdb
 ```
 
 The following runs `elicit` on the Cartesian product of the listed models and datasets, storing it in a special folder
-ELK_DIR/sweeps/<memorable_name>. Moreover, `--add_pooled` adds an additional dataset that pools all of the datasets
+CCS_DIR/sweeps/<memorable_name>. Moreover, `--add_pooled` adds an additional dataset that pools all of the datasets
 together. You can also add a `--visualize` flag to visualize the results of the sweep.
 
 ```bash
-elk sweep --models gpt2-{medium,large,xl} --datasets imdb amazon_polarity --add_pooled
+ccs sweep --models gpt2-{medium,large,xl} --datasets imdb amazon_polarity --add_pooled
 ```
 
-If you just do `elk plot`, it will plot the results from the most recent sweep.
+If you just do `ccs plot`, it will plot the results from the most recent sweep.
 If you want to plot a specific sweep, you can do so with:
 
 ```bash
-elk plot {sweep_name}
+ccs plot {sweep_name}
 ```
 
 ## Caching
 
-The hidden states resulting from `elk elicit` are cached as a HuggingFace dataset to avoid having to recompute them
+The hidden states resulting from `ccs elicit` are cached as a HuggingFace dataset to avoid having to recompute them
 every time we want to train a probe. The cache is stored in the same place as all other HuggingFace datasets, which is
 usually `~/.cache/huggingface/datasets`.
 
@@ -81,7 +81,7 @@ Use `pip install pre-commit && pre-commit install` in the root folder before you
 https://img.shields.io/static/v1?label=Remote%20-%20Containers&message=Open&color=blue&logo=visualstudiocode
 )
 ](
-https://vscode.dev/redirect?url=vscode://ms-vscode-remote.remote-containers/cloneInVolume?url=https://github.com/EleutherAI/elk
+https://vscode.dev/redirect?url=vscode://ms-vscode-remote.remote-containers/cloneInVolume?url=https://github.com/EleutherAI/ccs
 )
 
 ### Run tests
diff --git a/elk/__init__.py b/ccs/__init__.py
similarity index 100%
rename from elk/__init__.py
rename to ccs/__init__.py
diff --git a/elk/__main__.py b/ccs/__main__.py
similarity index 71%
rename from elk/__main__.py
rename to ccs/__main__.py
index 86e5087a..b1caef9b 100644
--- a/elk/__main__.py
+++ b/ccs/__main__.py
@@ -1,13 +1,13 @@
-"""Main entry point for `elk`."""
+"""Main entry point for `ccs`."""
 
 from dataclasses import dataclass
 
 from simple_parsing import ArgumentParser
 
-from elk.evaluation.evaluate import Eval
-from elk.plotting.command import Plot
-from elk.training.sweep import Sweep
-from elk.training.train import Elicit
+from ccs.evaluation.evaluate import Eval
+from ccs.plotting.command import Plot
+from ccs.training.sweep import Sweep
+from ccs.training.train import Elicit
 
 
 @dataclass
diff --git a/elk/debug_logging.py b/ccs/debug_logging.py
similarity index 100%
rename from elk/debug_logging.py
rename to ccs/debug_logging.py
diff --git a/elk/evaluation/__init__.py b/ccs/evaluation/__init__.py
similarity index 100%
rename from elk/evaluation/__init__.py
rename to ccs/evaluation/__init__.py
diff --git a/elk/evaluation/evaluate.py b/ccs/evaluation/evaluate.py
similarity index 95%
rename from elk/evaluation/evaluate.py
rename to ccs/evaluation/evaluate.py
index 4e6ec59f..53df3715 100644
--- a/elk/evaluation/evaluate.py
+++ b/ccs/evaluation/evaluate.py
@@ -6,7 +6,7 @@
 import torch
 from simple_parsing.helpers import field
 
-from ..files import elk_reporter_dir
+from ..files import ccs_reporter_dir
 from ..metrics import evaluate_preds
 from ..run import Run
 from ..utils import Color
@@ -22,7 +22,7 @@ class Eval(Run):
     def __post_init__(self):
         # Set our output directory before super().execute() does
         if not self.out_dir:
-            root = elk_reporter_dir() / self.source
+            root = ccs_reporter_dir() / self.source
             self.out_dir = root / "transfer" / "+".join(self.data.datasets)
 
     def execute(self, highlight_color: Color = "cyan"):
@@ -36,7 +36,7 @@ def apply_to_layer(
         device = self.get_device(devices, world_size)
         val_output = self.prepare_data(device, layer, "val")
 
-        experiment_dir = elk_reporter_dir() / self.source
+        experiment_dir = ccs_reporter_dir() / self.source
 
         reporter_path = experiment_dir / "reporters" / f"layer_{layer}.pt"
         reporter = torch.load(reporter_path, map_location=device)
diff --git a/elk/extraction/__init__.py b/ccs/extraction/__init__.py
similarity index 100%
rename from elk/extraction/__init__.py
rename to ccs/extraction/__init__.py
diff --git a/elk/extraction/balanced_sampler.py b/ccs/extraction/balanced_sampler.py
similarity index 100%
rename from elk/extraction/balanced_sampler.py
rename to ccs/extraction/balanced_sampler.py
diff --git a/elk/extraction/dataset_name.py b/ccs/extraction/dataset_name.py
similarity index 100%
rename from elk/extraction/dataset_name.py
rename to ccs/extraction/dataset_name.py
diff --git a/elk/extraction/extraction.py b/ccs/extraction/extraction.py
similarity index 100%
rename from elk/extraction/extraction.py
rename to ccs/extraction/extraction.py
diff --git a/elk/extraction/generator.py b/ccs/extraction/generator.py
similarity index 100%
rename from elk/extraction/generator.py
rename to ccs/extraction/generator.py
diff --git a/elk/extraction/prompt_loading.py b/ccs/extraction/prompt_loading.py
similarity index 100%
rename from elk/extraction/prompt_loading.py
rename to ccs/extraction/prompt_loading.py
diff --git a/elk/files.py b/ccs/files.py
similarity index 84%
rename from elk/files.py
rename to ccs/files.py
index 8546d4ab..96d5dd88 100644
--- a/elk/files.py
+++ b/ccs/files.py
@@ -8,14 +8,14 @@
 
 def sweeps_dir() -> Path:
     """Return the directory where sweeps are stored."""
-    return elk_reporter_dir() / "sweeps"
+    return ccs_reporter_dir() / "sweeps"
 
 
-def elk_reporter_dir() -> Path:
+def ccs_reporter_dir() -> Path:
     """Return the directory where reporter checkpoints and logs are stored."""
-    env_dir = os.environ.get("ELK_DIR", None)
+    env_dir = os.environ.get("CCS_DIR", None)
     if env_dir is None:
-        log_dir = Path.home() / "elk-reporters"
+        log_dir = Path.home() / "ccs-reporters"
     else:
         log_dir = Path(env_dir)
 
@@ -47,4 +47,4 @@ def memorably_named_dir(parent: Path):
 
 def transfer_eval_directory(source: str) -> Path:
     """Return the directory where transfer evals are stored."""
-    return elk_reporter_dir() / source / "transfer_eval"
+    return ccs_reporter_dir() / source / "transfer_eval"
diff --git a/elk/metrics/__init__.py b/ccs/metrics/__init__.py
similarity index 100%
rename from elk/metrics/__init__.py
rename to ccs/metrics/__init__.py
diff --git a/elk/metrics/accuracy.py b/ccs/metrics/accuracy.py
similarity index 100%
rename from elk/metrics/accuracy.py
rename to ccs/metrics/accuracy.py
diff --git a/elk/metrics/calibration.py b/ccs/metrics/calibration.py
similarity index 100%
rename from elk/metrics/calibration.py
rename to ccs/metrics/calibration.py
diff --git a/elk/metrics/eval.py b/ccs/metrics/eval.py
similarity index 100%
rename from elk/metrics/eval.py
rename to ccs/metrics/eval.py
diff --git a/elk/metrics/roc_auc.py b/ccs/metrics/roc_auc.py
similarity index 100%
rename from elk/metrics/roc_auc.py
rename to ccs/metrics/roc_auc.py
diff --git a/elk/parsing.py b/ccs/parsing.py
similarity index 100%
rename from elk/parsing.py
rename to ccs/parsing.py
diff --git a/elk/plotting/command.py b/ccs/plotting/command.py
similarity index 100%
rename from elk/plotting/command.py
rename to ccs/plotting/command.py
diff --git a/elk/plotting/visualize.py b/ccs/plotting/visualize.py
similarity index 100%
rename from elk/plotting/visualize.py
rename to ccs/plotting/visualize.py
diff --git a/elk/promptsource/LICENSE b/ccs/promptsource/LICENSE
similarity index 100%
rename from elk/promptsource/LICENSE
rename to ccs/promptsource/LICENSE
diff --git a/elk/promptsource/__init__.py b/ccs/promptsource/__init__.py
similarity index 100%
rename from elk/promptsource/__init__.py
rename to ccs/promptsource/__init__.py
diff --git a/elk/promptsource/templates.py b/ccs/promptsource/templates.py
similarity index 100%
rename from elk/promptsource/templates.py
rename to ccs/promptsource/templates.py
diff --git a/elk/promptsource/templates/Zaid/coqa_expanded/templates.yaml b/ccs/promptsource/templates/Zaid/coqa_expanded/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/Zaid/coqa_expanded/templates.yaml
rename to ccs/promptsource/templates/Zaid/coqa_expanded/templates.yaml
diff --git a/elk/promptsource/templates/Zaid/quac_expanded/templates.yaml b/ccs/promptsource/templates/Zaid/quac_expanded/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/Zaid/quac_expanded/templates.yaml
rename to ccs/promptsource/templates/Zaid/quac_expanded/templates.yaml
diff --git a/elk/promptsource/templates/acronym_identification/templates.yaml b/ccs/promptsource/templates/acronym_identification/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/acronym_identification/templates.yaml
rename to ccs/promptsource/templates/acronym_identification/templates.yaml
diff --git a/elk/promptsource/templates/ade_corpus_v2/Ade_corpus_v2_classification/templates.yaml b/ccs/promptsource/templates/ade_corpus_v2/Ade_corpus_v2_classification/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/ade_corpus_v2/Ade_corpus_v2_classification/templates.yaml
rename to ccs/promptsource/templates/ade_corpus_v2/Ade_corpus_v2_classification/templates.yaml
diff --git a/elk/promptsource/templates/ade_corpus_v2/Ade_corpus_v2_drug_ade_relation/templates.yaml b/ccs/promptsource/templates/ade_corpus_v2/Ade_corpus_v2_drug_ade_relation/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/ade_corpus_v2/Ade_corpus_v2_drug_ade_relation/templates.yaml
rename to ccs/promptsource/templates/ade_corpus_v2/Ade_corpus_v2_drug_ade_relation/templates.yaml
diff --git a/elk/promptsource/templates/ade_corpus_v2/Ade_corpus_v2_drug_dosage_relation/templates.yaml b/ccs/promptsource/templates/ade_corpus_v2/Ade_corpus_v2_drug_dosage_relation/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/ade_corpus_v2/Ade_corpus_v2_drug_dosage_relation/templates.yaml
rename to ccs/promptsource/templates/ade_corpus_v2/Ade_corpus_v2_drug_dosage_relation/templates.yaml
diff --git a/elk/promptsource/templates/adversarial_qa/adversarialQA/templates.yaml b/ccs/promptsource/templates/adversarial_qa/adversarialQA/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/adversarial_qa/adversarialQA/templates.yaml
rename to ccs/promptsource/templates/adversarial_qa/adversarialQA/templates.yaml
diff --git a/elk/promptsource/templates/adversarial_qa/dbert/templates.yaml b/ccs/promptsource/templates/adversarial_qa/dbert/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/adversarial_qa/dbert/templates.yaml
rename to ccs/promptsource/templates/adversarial_qa/dbert/templates.yaml
diff --git a/elk/promptsource/templates/adversarial_qa/dbidaf/templates.yaml b/ccs/promptsource/templates/adversarial_qa/dbidaf/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/adversarial_qa/dbidaf/templates.yaml
rename to ccs/promptsource/templates/adversarial_qa/dbidaf/templates.yaml
diff --git a/elk/promptsource/templates/adversarial_qa/droberta/templates.yaml b/ccs/promptsource/templates/adversarial_qa/droberta/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/adversarial_qa/droberta/templates.yaml
rename to ccs/promptsource/templates/adversarial_qa/droberta/templates.yaml
diff --git a/elk/promptsource/templates/aeslc/templates.yaml b/ccs/promptsource/templates/aeslc/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/aeslc/templates.yaml
rename to ccs/promptsource/templates/aeslc/templates.yaml
diff --git a/elk/promptsource/templates/ag_news/templates.yaml b/ccs/promptsource/templates/ag_news/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/ag_news/templates.yaml
rename to ccs/promptsource/templates/ag_news/templates.yaml
diff --git a/elk/promptsource/templates/ai2_arc/ARC-Challenge/templates.yaml b/ccs/promptsource/templates/ai2_arc/ARC-Challenge/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/ai2_arc/ARC-Challenge/templates.yaml
rename to ccs/promptsource/templates/ai2_arc/ARC-Challenge/templates.yaml
diff --git a/elk/promptsource/templates/ai2_arc/ARC-Easy/templates.yaml b/ccs/promptsource/templates/ai2_arc/ARC-Easy/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/ai2_arc/ARC-Easy/templates.yaml
rename to ccs/promptsource/templates/ai2_arc/ARC-Easy/templates.yaml
diff --git a/elk/promptsource/templates/amazon_polarity/templates.yaml b/ccs/promptsource/templates/amazon_polarity/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/amazon_polarity/templates.yaml
rename to ccs/promptsource/templates/amazon_polarity/templates.yaml
diff --git a/elk/promptsource/templates/amazon_reviews_multi/en/templates.yaml b/ccs/promptsource/templates/amazon_reviews_multi/en/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/amazon_reviews_multi/en/templates.yaml
rename to ccs/promptsource/templates/amazon_reviews_multi/en/templates.yaml
diff --git a/elk/promptsource/templates/amazon_us_reviews/Wireless_v1_00/templates.yaml b/ccs/promptsource/templates/amazon_us_reviews/Wireless_v1_00/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/amazon_us_reviews/Wireless_v1_00/templates.yaml
rename to ccs/promptsource/templates/amazon_us_reviews/Wireless_v1_00/templates.yaml
diff --git a/elk/promptsource/templates/ambig_qa/light/templates.yaml b/ccs/promptsource/templates/ambig_qa/light/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/ambig_qa/light/templates.yaml
rename to ccs/promptsource/templates/ambig_qa/light/templates.yaml
diff --git a/elk/promptsource/templates/anli/templates.yaml b/ccs/promptsource/templates/anli/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/anli/templates.yaml
rename to ccs/promptsource/templates/anli/templates.yaml
diff --git a/elk/promptsource/templates/app_reviews/templates.yaml b/ccs/promptsource/templates/app_reviews/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/app_reviews/templates.yaml
rename to ccs/promptsource/templates/app_reviews/templates.yaml
diff --git a/elk/promptsource/templates/aqua_rat/raw/templates.yaml b/ccs/promptsource/templates/aqua_rat/raw/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/aqua_rat/raw/templates.yaml
rename to ccs/promptsource/templates/aqua_rat/raw/templates.yaml
diff --git a/elk/promptsource/templates/art/templates.yaml b/ccs/promptsource/templates/art/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/art/templates.yaml
rename to ccs/promptsource/templates/art/templates.yaml
diff --git a/elk/promptsource/templates/asnq/templates.yaml b/ccs/promptsource/templates/asnq/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/asnq/templates.yaml
rename to ccs/promptsource/templates/asnq/templates.yaml
diff --git a/elk/promptsource/templates/asset/ratings/templates.yaml b/ccs/promptsource/templates/asset/ratings/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/asset/ratings/templates.yaml
rename to ccs/promptsource/templates/asset/ratings/templates.yaml
diff --git a/elk/promptsource/templates/asset/simplification/templates.yaml b/ccs/promptsource/templates/asset/simplification/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/asset/simplification/templates.yaml
rename to ccs/promptsource/templates/asset/simplification/templates.yaml
diff --git a/elk/promptsource/templates/banking77/templates.yaml b/ccs/promptsource/templates/banking77/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/banking77/templates.yaml
rename to ccs/promptsource/templates/banking77/templates.yaml
diff --git a/elk/promptsource/templates/billsum/templates.yaml b/ccs/promptsource/templates/billsum/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/billsum/templates.yaml
rename to ccs/promptsource/templates/billsum/templates.yaml
diff --git a/elk/promptsource/templates/bing_coronavirus_query_set/templates.yaml b/ccs/promptsource/templates/bing_coronavirus_query_set/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/bing_coronavirus_query_set/templates.yaml
rename to ccs/promptsource/templates/bing_coronavirus_query_set/templates.yaml
diff --git a/elk/promptsource/templates/biosses/templates.yaml b/ccs/promptsource/templates/biosses/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/biosses/templates.yaml
rename to ccs/promptsource/templates/biosses/templates.yaml
diff --git a/elk/promptsource/templates/blbooksgenre/title_genre_classifiction/templates.yaml b/ccs/promptsource/templates/blbooksgenre/title_genre_classifiction/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/blbooksgenre/title_genre_classifiction/templates.yaml
rename to ccs/promptsource/templates/blbooksgenre/title_genre_classifiction/templates.yaml
diff --git a/elk/promptsource/templates/blended_skill_talk/templates.yaml b/ccs/promptsource/templates/blended_skill_talk/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/blended_skill_talk/templates.yaml
rename to ccs/promptsource/templates/blended_skill_talk/templates.yaml
diff --git a/elk/promptsource/templates/boolq_pt/templates.yaml b/ccs/promptsource/templates/boolq_pt/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/boolq_pt/templates.yaml
rename to ccs/promptsource/templates/boolq_pt/templates.yaml
diff --git a/elk/promptsource/templates/cbt/CN/templates.yaml b/ccs/promptsource/templates/cbt/CN/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/cbt/CN/templates.yaml
rename to ccs/promptsource/templates/cbt/CN/templates.yaml
diff --git a/elk/promptsource/templates/cbt/NE/templates.yaml b/ccs/promptsource/templates/cbt/NE/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/cbt/NE/templates.yaml
rename to ccs/promptsource/templates/cbt/NE/templates.yaml
diff --git a/elk/promptsource/templates/cbt/P/templates.yaml b/ccs/promptsource/templates/cbt/P/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/cbt/P/templates.yaml
rename to ccs/promptsource/templates/cbt/P/templates.yaml
diff --git a/elk/promptsource/templates/cbt/V/templates.yaml b/ccs/promptsource/templates/cbt/V/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/cbt/V/templates.yaml
rename to ccs/promptsource/templates/cbt/V/templates.yaml
diff --git a/elk/promptsource/templates/cbt/raw/templates.yaml b/ccs/promptsource/templates/cbt/raw/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/cbt/raw/templates.yaml
rename to ccs/promptsource/templates/cbt/raw/templates.yaml
diff --git a/elk/promptsource/templates/cc_news/templates.yaml b/ccs/promptsource/templates/cc_news/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/cc_news/templates.yaml
rename to ccs/promptsource/templates/cc_news/templates.yaml
diff --git a/elk/promptsource/templates/christykoh/ag_news_pt/templates.yaml b/ccs/promptsource/templates/christykoh/ag_news_pt/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/christykoh/ag_news_pt/templates.yaml
rename to ccs/promptsource/templates/christykoh/ag_news_pt/templates.yaml
diff --git a/elk/promptsource/templates/christykoh/boolq_pt/templates.yaml b/ccs/promptsource/templates/christykoh/boolq_pt/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/christykoh/boolq_pt/templates.yaml
rename to ccs/promptsource/templates/christykoh/boolq_pt/templates.yaml
diff --git a/elk/promptsource/templates/christykoh/imdb_ar/templates.yaml b/ccs/promptsource/templates/christykoh/imdb_ar/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/christykoh/imdb_ar/templates.yaml
rename to ccs/promptsource/templates/christykoh/imdb_ar/templates.yaml
diff --git a/elk/promptsource/templates/christykoh/imdb_es/templates.yaml b/ccs/promptsource/templates/christykoh/imdb_es/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/christykoh/imdb_es/templates.yaml
rename to ccs/promptsource/templates/christykoh/imdb_es/templates.yaml
diff --git a/elk/promptsource/templates/christykoh/imdb_fr/templates.yaml b/ccs/promptsource/templates/christykoh/imdb_fr/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/christykoh/imdb_fr/templates.yaml
rename to ccs/promptsource/templates/christykoh/imdb_fr/templates.yaml
diff --git a/elk/promptsource/templates/christykoh/imdb_pt/templates.yaml b/ccs/promptsource/templates/christykoh/imdb_pt/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/christykoh/imdb_pt/templates.yaml
rename to ccs/promptsource/templates/christykoh/imdb_pt/templates.yaml
diff --git a/elk/promptsource/templates/christykoh/imdb_zh/templates.yaml b/ccs/promptsource/templates/christykoh/imdb_zh/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/christykoh/imdb_zh/templates.yaml
rename to ccs/promptsource/templates/christykoh/imdb_zh/templates.yaml
diff --git a/elk/promptsource/templates/circa/templates.yaml b/ccs/promptsource/templates/circa/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/circa/templates.yaml
rename to ccs/promptsource/templates/circa/templates.yaml
diff --git a/elk/promptsource/templates/climate_fever/templates.yaml b/ccs/promptsource/templates/climate_fever/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/climate_fever/templates.yaml
rename to ccs/promptsource/templates/climate_fever/templates.yaml
diff --git a/elk/promptsource/templates/cnn_dailymail/3.0.0/templates.yaml b/ccs/promptsource/templates/cnn_dailymail/3.0.0/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/cnn_dailymail/3.0.0/templates.yaml
rename to ccs/promptsource/templates/cnn_dailymail/3.0.0/templates.yaml
diff --git a/elk/promptsource/templates/codah/codah/templates.yaml b/ccs/promptsource/templates/codah/codah/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/codah/codah/templates.yaml
rename to ccs/promptsource/templates/codah/codah/templates.yaml
diff --git a/elk/promptsource/templates/codah/fold_0/templates.yaml b/ccs/promptsource/templates/codah/fold_0/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/codah/fold_0/templates.yaml
rename to ccs/promptsource/templates/codah/fold_0/templates.yaml
diff --git a/elk/promptsource/templates/codah/fold_1/templates.yaml b/ccs/promptsource/templates/codah/fold_1/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/codah/fold_1/templates.yaml
rename to ccs/promptsource/templates/codah/fold_1/templates.yaml
diff --git a/elk/promptsource/templates/codah/fold_2/templates.yaml b/ccs/promptsource/templates/codah/fold_2/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/codah/fold_2/templates.yaml
rename to ccs/promptsource/templates/codah/fold_2/templates.yaml
diff --git a/elk/promptsource/templates/codah/fold_3/templates.yaml b/ccs/promptsource/templates/codah/fold_3/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/codah/fold_3/templates.yaml
rename to ccs/promptsource/templates/codah/fold_3/templates.yaml
diff --git a/elk/promptsource/templates/codah/fold_4/templates.yaml b/ccs/promptsource/templates/codah/fold_4/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/codah/fold_4/templates.yaml
rename to ccs/promptsource/templates/codah/fold_4/templates.yaml
diff --git a/elk/promptsource/templates/code_x_glue_tc_text_to_code/templates.yaml b/ccs/promptsource/templates/code_x_glue_tc_text_to_code/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/code_x_glue_tc_text_to_code/templates.yaml
rename to ccs/promptsource/templates/code_x_glue_tc_text_to_code/templates.yaml
diff --git a/elk/promptsource/templates/common_gen/templates.yaml b/ccs/promptsource/templates/common_gen/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/common_gen/templates.yaml
rename to ccs/promptsource/templates/common_gen/templates.yaml
diff --git a/elk/promptsource/templates/commonsense_qa/templates.yaml b/ccs/promptsource/templates/commonsense_qa/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/commonsense_qa/templates.yaml
rename to ccs/promptsource/templates/commonsense_qa/templates.yaml
diff --git a/elk/promptsource/templates/conv_ai/templates.yaml b/ccs/promptsource/templates/conv_ai/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/conv_ai/templates.yaml
rename to ccs/promptsource/templates/conv_ai/templates.yaml
diff --git a/elk/promptsource/templates/conv_ai_2/templates.yaml b/ccs/promptsource/templates/conv_ai_2/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/conv_ai_2/templates.yaml
rename to ccs/promptsource/templates/conv_ai_2/templates.yaml
diff --git a/elk/promptsource/templates/conv_ai_3/templates.yaml b/ccs/promptsource/templates/conv_ai_3/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/conv_ai_3/templates.yaml
rename to ccs/promptsource/templates/conv_ai_3/templates.yaml
diff --git a/elk/promptsource/templates/coqa/templates.yaml b/ccs/promptsource/templates/coqa/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/coqa/templates.yaml
rename to ccs/promptsource/templates/coqa/templates.yaml
diff --git a/elk/promptsource/templates/cord19/metadata/templates.yaml b/ccs/promptsource/templates/cord19/metadata/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/cord19/metadata/templates.yaml
rename to ccs/promptsource/templates/cord19/metadata/templates.yaml
diff --git a/elk/promptsource/templates/cos_e/v1.0/templates.yaml b/ccs/promptsource/templates/cos_e/v1.0/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/cos_e/v1.0/templates.yaml
rename to ccs/promptsource/templates/cos_e/v1.0/templates.yaml
diff --git a/elk/promptsource/templates/cos_e/v1.11/templates.yaml b/ccs/promptsource/templates/cos_e/v1.11/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/cos_e/v1.11/templates.yaml
rename to ccs/promptsource/templates/cos_e/v1.11/templates.yaml
diff --git a/elk/promptsource/templates/cosmos_qa/templates.yaml b/ccs/promptsource/templates/cosmos_qa/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/cosmos_qa/templates.yaml
rename to ccs/promptsource/templates/cosmos_qa/templates.yaml
diff --git a/elk/promptsource/templates/covid_qa_castorini/templates.yaml b/ccs/promptsource/templates/covid_qa_castorini/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/covid_qa_castorini/templates.yaml
rename to ccs/promptsource/templates/covid_qa_castorini/templates.yaml
diff --git a/elk/promptsource/templates/craffel/openai_lambada/templates.yaml b/ccs/promptsource/templates/craffel/openai_lambada/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/craffel/openai_lambada/templates.yaml
rename to ccs/promptsource/templates/craffel/openai_lambada/templates.yaml
diff --git a/elk/promptsource/templates/craigslist_bargains/templates.yaml b/ccs/promptsource/templates/craigslist_bargains/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/craigslist_bargains/templates.yaml
rename to ccs/promptsource/templates/craigslist_bargains/templates.yaml
diff --git a/elk/promptsource/templates/crows_pairs/templates.yaml b/ccs/promptsource/templates/crows_pairs/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/crows_pairs/templates.yaml
rename to ccs/promptsource/templates/crows_pairs/templates.yaml
diff --git a/elk/promptsource/templates/dbpedia_14/templates.yaml b/ccs/promptsource/templates/dbpedia_14/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/dbpedia_14/templates.yaml
rename to ccs/promptsource/templates/dbpedia_14/templates.yaml
diff --git a/elk/promptsource/templates/discofuse/discofuse-sport/templates.yaml b/ccs/promptsource/templates/discofuse/discofuse-sport/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/discofuse/discofuse-sport/templates.yaml
rename to ccs/promptsource/templates/discofuse/discofuse-sport/templates.yaml
diff --git a/elk/promptsource/templates/discofuse/discofuse-wikipedia/templates.yaml b/ccs/promptsource/templates/discofuse/discofuse-wikipedia/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/discofuse/discofuse-wikipedia/templates.yaml
rename to ccs/promptsource/templates/discofuse/discofuse-wikipedia/templates.yaml
diff --git a/elk/promptsource/templates/discovery/discovery/templates.yaml b/ccs/promptsource/templates/discovery/discovery/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/discovery/discovery/templates.yaml
rename to ccs/promptsource/templates/discovery/discovery/templates.yaml
diff --git a/elk/promptsource/templates/docred/templates.yaml b/ccs/promptsource/templates/docred/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/docred/templates.yaml
rename to ccs/promptsource/templates/docred/templates.yaml
diff --git a/elk/promptsource/templates/dream/templates.yaml b/ccs/promptsource/templates/dream/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/dream/templates.yaml
rename to ccs/promptsource/templates/dream/templates.yaml
diff --git a/elk/promptsource/templates/drop/templates.yaml b/ccs/promptsource/templates/drop/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/drop/templates.yaml
rename to ccs/promptsource/templates/drop/templates.yaml
diff --git a/elk/promptsource/templates/duorc/ParaphraseRC/templates.yaml b/ccs/promptsource/templates/duorc/ParaphraseRC/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/duorc/ParaphraseRC/templates.yaml
rename to ccs/promptsource/templates/duorc/ParaphraseRC/templates.yaml
diff --git a/elk/promptsource/templates/duorc/SelfRC/templates.yaml b/ccs/promptsource/templates/duorc/SelfRC/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/duorc/SelfRC/templates.yaml
rename to ccs/promptsource/templates/duorc/SelfRC/templates.yaml
diff --git a/elk/promptsource/templates/e2e_nlg_cleaned/templates.yaml b/ccs/promptsource/templates/e2e_nlg_cleaned/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/e2e_nlg_cleaned/templates.yaml
rename to ccs/promptsource/templates/e2e_nlg_cleaned/templates.yaml
diff --git a/elk/promptsource/templates/ecthr_cases/alleged-violation-prediction/templates.yaml b/ccs/promptsource/templates/ecthr_cases/alleged-violation-prediction/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/ecthr_cases/alleged-violation-prediction/templates.yaml
rename to ccs/promptsource/templates/ecthr_cases/alleged-violation-prediction/templates.yaml
diff --git a/elk/promptsource/templates/emo/templates.yaml b/ccs/promptsource/templates/emo/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/emo/templates.yaml
rename to ccs/promptsource/templates/emo/templates.yaml
diff --git a/elk/promptsource/templates/emotion/templates.yaml b/ccs/promptsource/templates/emotion/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/emotion/templates.yaml
rename to ccs/promptsource/templates/emotion/templates.yaml
diff --git a/elk/promptsource/templates/enriched_web_nlg/en/templates.yaml b/ccs/promptsource/templates/enriched_web_nlg/en/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/enriched_web_nlg/en/templates.yaml
rename to ccs/promptsource/templates/enriched_web_nlg/en/templates.yaml
diff --git a/elk/promptsource/templates/esnli/templates.yaml b/ccs/promptsource/templates/esnli/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/esnli/templates.yaml
rename to ccs/promptsource/templates/esnli/templates.yaml
diff --git a/elk/promptsource/templates/evidence_infer_treatment/1.1/templates.yaml b/ccs/promptsource/templates/evidence_infer_treatment/1.1/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/evidence_infer_treatment/1.1/templates.yaml
rename to ccs/promptsource/templates/evidence_infer_treatment/1.1/templates.yaml
diff --git a/elk/promptsource/templates/evidence_infer_treatment/2.0/templates.yaml b/ccs/promptsource/templates/evidence_infer_treatment/2.0/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/evidence_infer_treatment/2.0/templates.yaml
rename to ccs/promptsource/templates/evidence_infer_treatment/2.0/templates.yaml
diff --git a/elk/promptsource/templates/fever/v1.0/templates.yaml b/ccs/promptsource/templates/fever/v1.0/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/fever/v1.0/templates.yaml
rename to ccs/promptsource/templates/fever/v1.0/templates.yaml
diff --git a/elk/promptsource/templates/fever/v2.0/templates.yaml b/ccs/promptsource/templates/fever/v2.0/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/fever/v2.0/templates.yaml
rename to ccs/promptsource/templates/fever/v2.0/templates.yaml
diff --git a/elk/promptsource/templates/financial_phrasebank/sentences_allagree/templates.yaml b/ccs/promptsource/templates/financial_phrasebank/sentences_allagree/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/financial_phrasebank/sentences_allagree/templates.yaml
rename to ccs/promptsource/templates/financial_phrasebank/sentences_allagree/templates.yaml
diff --git a/elk/promptsource/templates/freebase_qa/templates.yaml b/ccs/promptsource/templates/freebase_qa/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/freebase_qa/templates.yaml
rename to ccs/promptsource/templates/freebase_qa/templates.yaml
diff --git a/elk/promptsource/templates/generated_reviews_enth/templates.yaml b/ccs/promptsource/templates/generated_reviews_enth/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/generated_reviews_enth/templates.yaml
rename to ccs/promptsource/templates/generated_reviews_enth/templates.yaml
diff --git a/elk/promptsource/templates/gigaword/templates.yaml b/ccs/promptsource/templates/gigaword/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/gigaword/templates.yaml
rename to ccs/promptsource/templates/gigaword/templates.yaml
diff --git a/elk/promptsource/templates/glue/ax/templates.yaml b/ccs/promptsource/templates/glue/ax/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/glue/ax/templates.yaml
rename to ccs/promptsource/templates/glue/ax/templates.yaml
diff --git a/elk/promptsource/templates/glue/cola/templates.yaml b/ccs/promptsource/templates/glue/cola/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/glue/cola/templates.yaml
rename to ccs/promptsource/templates/glue/cola/templates.yaml
diff --git a/elk/promptsource/templates/glue/mnli/templates.yaml b/ccs/promptsource/templates/glue/mnli/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/glue/mnli/templates.yaml
rename to ccs/promptsource/templates/glue/mnli/templates.yaml
diff --git a/elk/promptsource/templates/glue/mnli_matched/templates.yaml b/ccs/promptsource/templates/glue/mnli_matched/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/glue/mnli_matched/templates.yaml
rename to ccs/promptsource/templates/glue/mnli_matched/templates.yaml
diff --git a/elk/promptsource/templates/glue/mnli_mismatched/templates.yaml b/ccs/promptsource/templates/glue/mnli_mismatched/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/glue/mnli_mismatched/templates.yaml
rename to ccs/promptsource/templates/glue/mnli_mismatched/templates.yaml
diff --git a/elk/promptsource/templates/glue/mrpc/templates.yaml b/ccs/promptsource/templates/glue/mrpc/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/glue/mrpc/templates.yaml
rename to ccs/promptsource/templates/glue/mrpc/templates.yaml
diff --git a/elk/promptsource/templates/glue/qnli/templates.yaml b/ccs/promptsource/templates/glue/qnli/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/glue/qnli/templates.yaml
rename to ccs/promptsource/templates/glue/qnli/templates.yaml
diff --git a/elk/promptsource/templates/glue/qqp/templates.yaml b/ccs/promptsource/templates/glue/qqp/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/glue/qqp/templates.yaml
rename to ccs/promptsource/templates/glue/qqp/templates.yaml
diff --git a/elk/promptsource/templates/glue/rte/templates.yaml b/ccs/promptsource/templates/glue/rte/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/glue/rte/templates.yaml
rename to ccs/promptsource/templates/glue/rte/templates.yaml
diff --git a/elk/promptsource/templates/glue/sst2/templates.yaml b/ccs/promptsource/templates/glue/sst2/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/glue/sst2/templates.yaml
rename to ccs/promptsource/templates/glue/sst2/templates.yaml
diff --git a/elk/promptsource/templates/glue/stsb/templates.yaml b/ccs/promptsource/templates/glue/stsb/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/glue/stsb/templates.yaml
rename to ccs/promptsource/templates/glue/stsb/templates.yaml
diff --git a/elk/promptsource/templates/glue/wnli/templates.yaml b/ccs/promptsource/templates/glue/wnli/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/glue/wnli/templates.yaml
rename to ccs/promptsource/templates/glue/wnli/templates.yaml
diff --git a/elk/promptsource/templates/google_wellformed_query/templates.yaml b/ccs/promptsource/templates/google_wellformed_query/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/google_wellformed_query/templates.yaml
rename to ccs/promptsource/templates/google_wellformed_query/templates.yaml
diff --git a/elk/promptsource/templates/great_code/templates.yaml b/ccs/promptsource/templates/great_code/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/great_code/templates.yaml
rename to ccs/promptsource/templates/great_code/templates.yaml
diff --git a/elk/promptsource/templates/guardian_authorship/cross_genre_1/templates.yaml b/ccs/promptsource/templates/guardian_authorship/cross_genre_1/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/guardian_authorship/cross_genre_1/templates.yaml
rename to ccs/promptsource/templates/guardian_authorship/cross_genre_1/templates.yaml
diff --git a/elk/promptsource/templates/guardian_authorship/cross_topic_1/templates.yaml b/ccs/promptsource/templates/guardian_authorship/cross_topic_1/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/guardian_authorship/cross_topic_1/templates.yaml
rename to ccs/promptsource/templates/guardian_authorship/cross_topic_1/templates.yaml
diff --git a/elk/promptsource/templates/guardian_authorship/cross_topic_4/templates.yaml b/ccs/promptsource/templates/guardian_authorship/cross_topic_4/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/guardian_authorship/cross_topic_4/templates.yaml
rename to ccs/promptsource/templates/guardian_authorship/cross_topic_4/templates.yaml
diff --git a/elk/promptsource/templates/guardian_authorship/cross_topic_7/templates.yaml b/ccs/promptsource/templates/guardian_authorship/cross_topic_7/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/guardian_authorship/cross_topic_7/templates.yaml
rename to ccs/promptsource/templates/guardian_authorship/cross_topic_7/templates.yaml
diff --git a/elk/promptsource/templates/gutenberg_time/templates.yaml b/ccs/promptsource/templates/gutenberg_time/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/gutenberg_time/templates.yaml
rename to ccs/promptsource/templates/gutenberg_time/templates.yaml
diff --git a/elk/promptsource/templates/hans/templates.yaml b/ccs/promptsource/templates/hans/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/hans/templates.yaml
rename to ccs/promptsource/templates/hans/templates.yaml
diff --git a/elk/promptsource/templates/hate_speech18/templates.yaml b/ccs/promptsource/templates/hate_speech18/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/hate_speech18/templates.yaml
rename to ccs/promptsource/templates/hate_speech18/templates.yaml
diff --git a/elk/promptsource/templates/head_qa/en/templates.yaml b/ccs/promptsource/templates/head_qa/en/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/head_qa/en/templates.yaml
rename to ccs/promptsource/templates/head_qa/en/templates.yaml
diff --git a/elk/promptsource/templates/health_fact/templates.yaml b/ccs/promptsource/templates/health_fact/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/health_fact/templates.yaml
rename to ccs/promptsource/templates/health_fact/templates.yaml
diff --git a/elk/promptsource/templates/hellaswag/templates.yaml b/ccs/promptsource/templates/hellaswag/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/hellaswag/templates.yaml
rename to ccs/promptsource/templates/hellaswag/templates.yaml
diff --git a/elk/promptsource/templates/hlgd/templates.yaml b/ccs/promptsource/templates/hlgd/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/hlgd/templates.yaml
rename to ccs/promptsource/templates/hlgd/templates.yaml
diff --git a/elk/promptsource/templates/hotpot_qa/distractor/templates.yaml b/ccs/promptsource/templates/hotpot_qa/distractor/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/hotpot_qa/distractor/templates.yaml
rename to ccs/promptsource/templates/hotpot_qa/distractor/templates.yaml
diff --git a/elk/promptsource/templates/hotpot_qa/fullwiki/templates.yaml b/ccs/promptsource/templates/hotpot_qa/fullwiki/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/hotpot_qa/fullwiki/templates.yaml
rename to ccs/promptsource/templates/hotpot_qa/fullwiki/templates.yaml
diff --git a/elk/promptsource/templates/humicroedit/subtask-1/templates.yaml b/ccs/promptsource/templates/humicroedit/subtask-1/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/humicroedit/subtask-1/templates.yaml
rename to ccs/promptsource/templates/humicroedit/subtask-1/templates.yaml
diff --git a/elk/promptsource/templates/humicroedit/subtask-2/templates.yaml b/ccs/promptsource/templates/humicroedit/subtask-2/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/humicroedit/subtask-2/templates.yaml
rename to ccs/promptsource/templates/humicroedit/subtask-2/templates.yaml
diff --git a/elk/promptsource/templates/hyperpartisan_news_detection/byarticle/templates.yaml b/ccs/promptsource/templates/hyperpartisan_news_detection/byarticle/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/hyperpartisan_news_detection/byarticle/templates.yaml
rename to ccs/promptsource/templates/hyperpartisan_news_detection/byarticle/templates.yaml
diff --git a/elk/promptsource/templates/hyperpartisan_news_detection/bypublisher/templates.yaml b/ccs/promptsource/templates/hyperpartisan_news_detection/bypublisher/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/hyperpartisan_news_detection/bypublisher/templates.yaml
rename to ccs/promptsource/templates/hyperpartisan_news_detection/bypublisher/templates.yaml
diff --git a/elk/promptsource/templates/imdb/templates.yaml b/ccs/promptsource/templates/imdb/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/imdb/templates.yaml
rename to ccs/promptsource/templates/imdb/templates.yaml
diff --git a/elk/promptsource/templates/jfleg/templates.yaml b/ccs/promptsource/templates/jfleg/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/jfleg/templates.yaml
rename to ccs/promptsource/templates/jfleg/templates.yaml
diff --git a/elk/promptsource/templates/jigsaw_unintended_bias/templates.yaml b/ccs/promptsource/templates/jigsaw_unintended_bias/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/jigsaw_unintended_bias/templates.yaml
rename to ccs/promptsource/templates/jigsaw_unintended_bias/templates.yaml
diff --git a/elk/promptsource/templates/kelm/templates.yaml b/ccs/promptsource/templates/kelm/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/kelm/templates.yaml
rename to ccs/promptsource/templates/kelm/templates.yaml
diff --git a/elk/promptsource/templates/kilt_tasks/hotpotqa/templates.yaml b/ccs/promptsource/templates/kilt_tasks/hotpotqa/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/kilt_tasks/hotpotqa/templates.yaml
rename to ccs/promptsource/templates/kilt_tasks/hotpotqa/templates.yaml
diff --git a/elk/promptsource/templates/kilt_tasks/nq/templates.yaml b/ccs/promptsource/templates/kilt_tasks/nq/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/kilt_tasks/nq/templates.yaml
rename to ccs/promptsource/templates/kilt_tasks/nq/templates.yaml
diff --git a/elk/promptsource/templates/lama/trex/templates.yaml b/ccs/promptsource/templates/lama/trex/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/lama/trex/templates.yaml
rename to ccs/promptsource/templates/lama/trex/templates.yaml
diff --git a/elk/promptsource/templates/lambada/templates.yaml b/ccs/promptsource/templates/lambada/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/lambada/templates.yaml
rename to ccs/promptsource/templates/lambada/templates.yaml
diff --git a/elk/promptsource/templates/lauritowal/redefine_math/templates.yaml b/ccs/promptsource/templates/lauritowal/redefine_math/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/lauritowal/redefine_math/templates.yaml
rename to ccs/promptsource/templates/lauritowal/redefine_math/templates.yaml
diff --git a/elk/promptsource/templates/liar/templates.yaml b/ccs/promptsource/templates/liar/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/liar/templates.yaml
rename to ccs/promptsource/templates/liar/templates.yaml
diff --git a/elk/promptsource/templates/limit/templates.yaml b/ccs/promptsource/templates/limit/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/limit/templates.yaml
rename to ccs/promptsource/templates/limit/templates.yaml
diff --git a/elk/promptsource/templates/math_dataset/algebra__linear_1d/templates.yaml b/ccs/promptsource/templates/math_dataset/algebra__linear_1d/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/math_dataset/algebra__linear_1d/templates.yaml
rename to ccs/promptsource/templates/math_dataset/algebra__linear_1d/templates.yaml
diff --git a/elk/promptsource/templates/math_dataset/algebra__linear_1d_composed/templates.yaml b/ccs/promptsource/templates/math_dataset/algebra__linear_1d_composed/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/math_dataset/algebra__linear_1d_composed/templates.yaml
rename to ccs/promptsource/templates/math_dataset/algebra__linear_1d_composed/templates.yaml
diff --git a/elk/promptsource/templates/math_dataset/algebra__linear_2d/templates.yaml b/ccs/promptsource/templates/math_dataset/algebra__linear_2d/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/math_dataset/algebra__linear_2d/templates.yaml
rename to ccs/promptsource/templates/math_dataset/algebra__linear_2d/templates.yaml
diff --git a/elk/promptsource/templates/math_dataset/algebra__linear_2d_composed/templates.yaml b/ccs/promptsource/templates/math_dataset/algebra__linear_2d_composed/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/math_dataset/algebra__linear_2d_composed/templates.yaml
rename to ccs/promptsource/templates/math_dataset/algebra__linear_2d_composed/templates.yaml
diff --git a/elk/promptsource/templates/math_qa/templates.yaml b/ccs/promptsource/templates/math_qa/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/math_qa/templates.yaml
rename to ccs/promptsource/templates/math_qa/templates.yaml
diff --git a/elk/promptsource/templates/mc_taco/templates.yaml b/ccs/promptsource/templates/mc_taco/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/mc_taco/templates.yaml
rename to ccs/promptsource/templates/mc_taco/templates.yaml
diff --git a/elk/promptsource/templates/mdd/task1_qa/templates.yaml b/ccs/promptsource/templates/mdd/task1_qa/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/mdd/task1_qa/templates.yaml
rename to ccs/promptsource/templates/mdd/task1_qa/templates.yaml
diff --git a/elk/promptsource/templates/mdd/task2_recs/templates.yaml b/ccs/promptsource/templates/mdd/task2_recs/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/mdd/task2_recs/templates.yaml
rename to ccs/promptsource/templates/mdd/task2_recs/templates.yaml
diff --git a/elk/promptsource/templates/mdd/task3_qarecs/templates.yaml b/ccs/promptsource/templates/mdd/task3_qarecs/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/mdd/task3_qarecs/templates.yaml
rename to ccs/promptsource/templates/mdd/task3_qarecs/templates.yaml
diff --git a/elk/promptsource/templates/medal/templates.yaml b/ccs/promptsource/templates/medal/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/medal/templates.yaml
rename to ccs/promptsource/templates/medal/templates.yaml
diff --git a/elk/promptsource/templates/medical_questions_pairs/templates.yaml b/ccs/promptsource/templates/medical_questions_pairs/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/medical_questions_pairs/templates.yaml
rename to ccs/promptsource/templates/medical_questions_pairs/templates.yaml
diff --git a/elk/promptsource/templates/meta_woz/dialogues/templates.yaml b/ccs/promptsource/templates/meta_woz/dialogues/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/meta_woz/dialogues/templates.yaml
rename to ccs/promptsource/templates/meta_woz/dialogues/templates.yaml
diff --git a/elk/promptsource/templates/mocha/templates.yaml b/ccs/promptsource/templates/mocha/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/mocha/templates.yaml
rename to ccs/promptsource/templates/mocha/templates.yaml
diff --git a/elk/promptsource/templates/movie_rationales/templates.yaml b/ccs/promptsource/templates/movie_rationales/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/movie_rationales/templates.yaml
rename to ccs/promptsource/templates/movie_rationales/templates.yaml
diff --git a/elk/promptsource/templates/multi_news/templates.yaml b/ccs/promptsource/templates/multi_news/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/multi_news/templates.yaml
rename to ccs/promptsource/templates/multi_news/templates.yaml
diff --git a/elk/promptsource/templates/multi_nli/templates.yaml b/ccs/promptsource/templates/multi_nli/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/multi_nli/templates.yaml
rename to ccs/promptsource/templates/multi_nli/templates.yaml
diff --git a/elk/promptsource/templates/multi_x_science_sum/templates.yaml b/ccs/promptsource/templates/multi_x_science_sum/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/multi_x_science_sum/templates.yaml
rename to ccs/promptsource/templates/multi_x_science_sum/templates.yaml
diff --git a/elk/promptsource/templates/mwsc/templates.yaml b/ccs/promptsource/templates/mwsc/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/mwsc/templates.yaml
rename to ccs/promptsource/templates/mwsc/templates.yaml
diff --git a/elk/promptsource/templates/narrativeqa/templates.yaml b/ccs/promptsource/templates/narrativeqa/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/narrativeqa/templates.yaml
rename to ccs/promptsource/templates/narrativeqa/templates.yaml
diff --git a/elk/promptsource/templates/ncbi_disease/templates.yaml b/ccs/promptsource/templates/ncbi_disease/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/ncbi_disease/templates.yaml
rename to ccs/promptsource/templates/ncbi_disease/templates.yaml
diff --git a/elk/promptsource/templates/neural_code_search/evaluation_dataset/templates.yaml b/ccs/promptsource/templates/neural_code_search/evaluation_dataset/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/neural_code_search/evaluation_dataset/templates.yaml
rename to ccs/promptsource/templates/neural_code_search/evaluation_dataset/templates.yaml
diff --git a/elk/promptsource/templates/newspop/templates.yaml b/ccs/promptsource/templates/newspop/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/newspop/templates.yaml
rename to ccs/promptsource/templates/newspop/templates.yaml
diff --git a/elk/promptsource/templates/nlu_evaluation_data/templates.yaml b/ccs/promptsource/templates/nlu_evaluation_data/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/nlu_evaluation_data/templates.yaml
rename to ccs/promptsource/templates/nlu_evaluation_data/templates.yaml
diff --git a/elk/promptsource/templates/nq_open/templates.yaml b/ccs/promptsource/templates/nq_open/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/nq_open/templates.yaml
rename to ccs/promptsource/templates/nq_open/templates.yaml
diff --git a/elk/promptsource/templates/numer_sense/templates.yaml b/ccs/promptsource/templates/numer_sense/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/numer_sense/templates.yaml
rename to ccs/promptsource/templates/numer_sense/templates.yaml
diff --git a/elk/promptsource/templates/onestop_english/templates.yaml b/ccs/promptsource/templates/onestop_english/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/onestop_english/templates.yaml
rename to ccs/promptsource/templates/onestop_english/templates.yaml
diff --git a/elk/promptsource/templates/openai_humaneval/templates.yaml b/ccs/promptsource/templates/openai_humaneval/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/openai_humaneval/templates.yaml
rename to ccs/promptsource/templates/openai_humaneval/templates.yaml
diff --git a/elk/promptsource/templates/openbookqa/additional/templates.yaml b/ccs/promptsource/templates/openbookqa/additional/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/openbookqa/additional/templates.yaml
rename to ccs/promptsource/templates/openbookqa/additional/templates.yaml
diff --git a/elk/promptsource/templates/openbookqa/main/templates.yaml b/ccs/promptsource/templates/openbookqa/main/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/openbookqa/main/templates.yaml
rename to ccs/promptsource/templates/openbookqa/main/templates.yaml
diff --git a/elk/promptsource/templates/paws-x/en/templates.yaml b/ccs/promptsource/templates/paws-x/en/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/paws-x/en/templates.yaml
rename to ccs/promptsource/templates/paws-x/en/templates.yaml
diff --git a/elk/promptsource/templates/paws/labeled_final/templates.yaml b/ccs/promptsource/templates/paws/labeled_final/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/paws/labeled_final/templates.yaml
rename to ccs/promptsource/templates/paws/labeled_final/templates.yaml
diff --git a/elk/promptsource/templates/paws/labeled_swap/templates.yaml b/ccs/promptsource/templates/paws/labeled_swap/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/paws/labeled_swap/templates.yaml
rename to ccs/promptsource/templates/paws/labeled_swap/templates.yaml
diff --git a/elk/promptsource/templates/paws/unlabeled_final/templates.yaml b/ccs/promptsource/templates/paws/unlabeled_final/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/paws/unlabeled_final/templates.yaml
rename to ccs/promptsource/templates/paws/unlabeled_final/templates.yaml
diff --git a/elk/promptsource/templates/piqa/templates.yaml b/ccs/promptsource/templates/piqa/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/piqa/templates.yaml
rename to ccs/promptsource/templates/piqa/templates.yaml
diff --git a/elk/promptsource/templates/poem_sentiment/templates.yaml b/ccs/promptsource/templates/poem_sentiment/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/poem_sentiment/templates.yaml
rename to ccs/promptsource/templates/poem_sentiment/templates.yaml
diff --git a/elk/promptsource/templates/pubmed_qa/pqa_labeled/templates.yaml b/ccs/promptsource/templates/pubmed_qa/pqa_labeled/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/pubmed_qa/pqa_labeled/templates.yaml
rename to ccs/promptsource/templates/pubmed_qa/pqa_labeled/templates.yaml
diff --git a/elk/promptsource/templates/qa_srl/templates.yaml b/ccs/promptsource/templates/qa_srl/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/qa_srl/templates.yaml
rename to ccs/promptsource/templates/qa_srl/templates.yaml
diff --git a/elk/promptsource/templates/qa_zre/templates.yaml b/ccs/promptsource/templates/qa_zre/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/qa_zre/templates.yaml
rename to ccs/promptsource/templates/qa_zre/templates.yaml
diff --git a/elk/promptsource/templates/qasc/templates.yaml b/ccs/promptsource/templates/qasc/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/qasc/templates.yaml
rename to ccs/promptsource/templates/qasc/templates.yaml
diff --git a/elk/promptsource/templates/qed/templates.yaml b/ccs/promptsource/templates/qed/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/qed/templates.yaml
rename to ccs/promptsource/templates/qed/templates.yaml
diff --git a/elk/promptsource/templates/quac/templates.yaml b/ccs/promptsource/templates/quac/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/quac/templates.yaml
rename to ccs/promptsource/templates/quac/templates.yaml
diff --git a/elk/promptsource/templates/quail/templates.yaml b/ccs/promptsource/templates/quail/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/quail/templates.yaml
rename to ccs/promptsource/templates/quail/templates.yaml
diff --git a/elk/promptsource/templates/quarel/templates.yaml b/ccs/promptsource/templates/quarel/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/quarel/templates.yaml
rename to ccs/promptsource/templates/quarel/templates.yaml
diff --git a/elk/promptsource/templates/quartz/templates.yaml b/ccs/promptsource/templates/quartz/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/quartz/templates.yaml
rename to ccs/promptsource/templates/quartz/templates.yaml
diff --git a/elk/promptsource/templates/quora/templates.yaml b/ccs/promptsource/templates/quora/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/quora/templates.yaml
rename to ccs/promptsource/templates/quora/templates.yaml
diff --git a/elk/promptsource/templates/quoref/templates.yaml b/ccs/promptsource/templates/quoref/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/quoref/templates.yaml
rename to ccs/promptsource/templates/quoref/templates.yaml
diff --git a/elk/promptsource/templates/race/all/templates.yaml b/ccs/promptsource/templates/race/all/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/race/all/templates.yaml
rename to ccs/promptsource/templates/race/all/templates.yaml
diff --git a/elk/promptsource/templates/race/high/templates.yaml b/ccs/promptsource/templates/race/high/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/race/high/templates.yaml
rename to ccs/promptsource/templates/race/high/templates.yaml
diff --git a/elk/promptsource/templates/race/middle/templates.yaml b/ccs/promptsource/templates/race/middle/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/race/middle/templates.yaml
rename to ccs/promptsource/templates/race/middle/templates.yaml
diff --git a/elk/promptsource/templates/reaganjlee/boolq_ar/templates.yaml b/ccs/promptsource/templates/reaganjlee/boolq_ar/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/reaganjlee/boolq_ar/templates.yaml
rename to ccs/promptsource/templates/reaganjlee/boolq_ar/templates.yaml
diff --git a/elk/promptsource/templates/reaganjlee/boolq_es/templates.yaml b/ccs/promptsource/templates/reaganjlee/boolq_es/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/reaganjlee/boolq_es/templates.yaml
rename to ccs/promptsource/templates/reaganjlee/boolq_es/templates.yaml
diff --git a/elk/promptsource/templates/reaganjlee/boolq_fr/templates.yaml b/ccs/promptsource/templates/reaganjlee/boolq_fr/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/reaganjlee/boolq_fr/templates.yaml
rename to ccs/promptsource/templates/reaganjlee/boolq_fr/templates.yaml
diff --git a/elk/promptsource/templates/reaganjlee/boolq_pt/templates.yaml b/ccs/promptsource/templates/reaganjlee/boolq_pt/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/reaganjlee/boolq_pt/templates.yaml
rename to ccs/promptsource/templates/reaganjlee/boolq_pt/templates.yaml
diff --git a/elk/promptsource/templates/reaganjlee/boolq_zh/templates.yaml b/ccs/promptsource/templates/reaganjlee/boolq_zh/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/reaganjlee/boolq_zh/templates.yaml
rename to ccs/promptsource/templates/reaganjlee/boolq_zh/templates.yaml
diff --git a/elk/promptsource/templates/reaganjlee/truthful_qa_mc/templates.yaml b/ccs/promptsource/templates/reaganjlee/truthful_qa_mc/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/reaganjlee/truthful_qa_mc/templates.yaml
rename to ccs/promptsource/templates/reaganjlee/truthful_qa_mc/templates.yaml
diff --git a/elk/promptsource/templates/reaganjlee/truthful_qa_mc_ar/templates.yaml b/ccs/promptsource/templates/reaganjlee/truthful_qa_mc_ar/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/reaganjlee/truthful_qa_mc_ar/templates.yaml
rename to ccs/promptsource/templates/reaganjlee/truthful_qa_mc_ar/templates.yaml
diff --git a/elk/promptsource/templates/reaganjlee/truthful_qa_mc_es/templates.yaml b/ccs/promptsource/templates/reaganjlee/truthful_qa_mc_es/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/reaganjlee/truthful_qa_mc_es/templates.yaml
rename to ccs/promptsource/templates/reaganjlee/truthful_qa_mc_es/templates.yaml
diff --git a/elk/promptsource/templates/reaganjlee/truthful_qa_mc_fr/templates.yaml b/ccs/promptsource/templates/reaganjlee/truthful_qa_mc_fr/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/reaganjlee/truthful_qa_mc_fr/templates.yaml
rename to ccs/promptsource/templates/reaganjlee/truthful_qa_mc_fr/templates.yaml
diff --git a/elk/promptsource/templates/reaganjlee/truthful_qa_mc_pt/templates.yaml b/ccs/promptsource/templates/reaganjlee/truthful_qa_mc_pt/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/reaganjlee/truthful_qa_mc_pt/templates.yaml
rename to ccs/promptsource/templates/reaganjlee/truthful_qa_mc_pt/templates.yaml
diff --git a/elk/promptsource/templates/reaganjlee/truthful_qa_mc_zh/templates.yaml b/ccs/promptsource/templates/reaganjlee/truthful_qa_mc_zh/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/reaganjlee/truthful_qa_mc_zh/templates.yaml
rename to ccs/promptsource/templates/reaganjlee/truthful_qa_mc_zh/templates.yaml
diff --git a/elk/promptsource/templates/riddle_sense/templates.yaml b/ccs/promptsource/templates/riddle_sense/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/riddle_sense/templates.yaml
rename to ccs/promptsource/templates/riddle_sense/templates.yaml
diff --git a/elk/promptsource/templates/ropes/templates.yaml b/ccs/promptsource/templates/ropes/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/ropes/templates.yaml
rename to ccs/promptsource/templates/ropes/templates.yaml
diff --git a/elk/promptsource/templates/rotten_tomatoes/templates.yaml b/ccs/promptsource/templates/rotten_tomatoes/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/rotten_tomatoes/templates.yaml
rename to ccs/promptsource/templates/rotten_tomatoes/templates.yaml
diff --git a/elk/promptsource/templates/samsum/templates.yaml b/ccs/promptsource/templates/samsum/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/samsum/templates.yaml
rename to ccs/promptsource/templates/samsum/templates.yaml
diff --git a/elk/promptsource/templates/scan/addprim_jump/templates.yaml b/ccs/promptsource/templates/scan/addprim_jump/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/scan/addprim_jump/templates.yaml
rename to ccs/promptsource/templates/scan/addprim_jump/templates.yaml
diff --git a/elk/promptsource/templates/scan/addprim_turn_left/templates.yaml b/ccs/promptsource/templates/scan/addprim_turn_left/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/scan/addprim_turn_left/templates.yaml
rename to ccs/promptsource/templates/scan/addprim_turn_left/templates.yaml
diff --git a/elk/promptsource/templates/scan/filler_num0/templates.yaml b/ccs/promptsource/templates/scan/filler_num0/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/scan/filler_num0/templates.yaml
rename to ccs/promptsource/templates/scan/filler_num0/templates.yaml
diff --git a/elk/promptsource/templates/scan/filler_num1/templates.yaml b/ccs/promptsource/templates/scan/filler_num1/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/scan/filler_num1/templates.yaml
rename to ccs/promptsource/templates/scan/filler_num1/templates.yaml
diff --git a/elk/promptsource/templates/scan/filler_num2/templates.yaml b/ccs/promptsource/templates/scan/filler_num2/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/scan/filler_num2/templates.yaml
rename to ccs/promptsource/templates/scan/filler_num2/templates.yaml
diff --git a/elk/promptsource/templates/scan/filler_num3/templates.yaml b/ccs/promptsource/templates/scan/filler_num3/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/scan/filler_num3/templates.yaml
rename to ccs/promptsource/templates/scan/filler_num3/templates.yaml
diff --git a/elk/promptsource/templates/scan/length/templates.yaml b/ccs/promptsource/templates/scan/length/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/scan/length/templates.yaml
rename to ccs/promptsource/templates/scan/length/templates.yaml
diff --git a/elk/promptsource/templates/scan/simple/templates.yaml b/ccs/promptsource/templates/scan/simple/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/scan/simple/templates.yaml
rename to ccs/promptsource/templates/scan/simple/templates.yaml
diff --git a/elk/promptsource/templates/scan/template_around_right/templates.yaml b/ccs/promptsource/templates/scan/template_around_right/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/scan/template_around_right/templates.yaml
rename to ccs/promptsource/templates/scan/template_around_right/templates.yaml
diff --git a/elk/promptsource/templates/scan/template_jump_around_right/templates.yaml b/ccs/promptsource/templates/scan/template_jump_around_right/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/scan/template_jump_around_right/templates.yaml
rename to ccs/promptsource/templates/scan/template_jump_around_right/templates.yaml
diff --git a/elk/promptsource/templates/scan/template_opposite_right/templates.yaml b/ccs/promptsource/templates/scan/template_opposite_right/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/scan/template_opposite_right/templates.yaml
rename to ccs/promptsource/templates/scan/template_opposite_right/templates.yaml
diff --git a/elk/promptsource/templates/scan/template_right/templates.yaml b/ccs/promptsource/templates/scan/template_right/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/scan/template_right/templates.yaml
rename to ccs/promptsource/templates/scan/template_right/templates.yaml
diff --git a/elk/promptsource/templates/scicite/templates.yaml b/ccs/promptsource/templates/scicite/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/scicite/templates.yaml
rename to ccs/promptsource/templates/scicite/templates.yaml
diff --git a/elk/promptsource/templates/scientific_papers/arxiv/templates.yaml b/ccs/promptsource/templates/scientific_papers/arxiv/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/scientific_papers/arxiv/templates.yaml
rename to ccs/promptsource/templates/scientific_papers/arxiv/templates.yaml
diff --git a/elk/promptsource/templates/scientific_papers/pubmed/templates.yaml b/ccs/promptsource/templates/scientific_papers/pubmed/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/scientific_papers/pubmed/templates.yaml
rename to ccs/promptsource/templates/scientific_papers/pubmed/templates.yaml
diff --git a/elk/promptsource/templates/sciq/templates.yaml b/ccs/promptsource/templates/sciq/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/sciq/templates.yaml
rename to ccs/promptsource/templates/sciq/templates.yaml
diff --git a/elk/promptsource/templates/scitail/snli_format/templates.yaml b/ccs/promptsource/templates/scitail/snli_format/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/scitail/snli_format/templates.yaml
rename to ccs/promptsource/templates/scitail/snli_format/templates.yaml
diff --git a/elk/promptsource/templates/scitail/tsv_format/templates.yaml b/ccs/promptsource/templates/scitail/tsv_format/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/scitail/tsv_format/templates.yaml
rename to ccs/promptsource/templates/scitail/tsv_format/templates.yaml
diff --git a/elk/promptsource/templates/scitldr/Abstract/templates.yaml b/ccs/promptsource/templates/scitldr/Abstract/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/scitldr/Abstract/templates.yaml
rename to ccs/promptsource/templates/scitldr/Abstract/templates.yaml
diff --git a/elk/promptsource/templates/selqa/answer_selection_analysis/templates.yaml b/ccs/promptsource/templates/selqa/answer_selection_analysis/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/selqa/answer_selection_analysis/templates.yaml
rename to ccs/promptsource/templates/selqa/answer_selection_analysis/templates.yaml
diff --git a/elk/promptsource/templates/sem_eval_2010_task_8/templates.yaml b/ccs/promptsource/templates/sem_eval_2010_task_8/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/sem_eval_2010_task_8/templates.yaml
rename to ccs/promptsource/templates/sem_eval_2010_task_8/templates.yaml
diff --git a/elk/promptsource/templates/sem_eval_2014_task_1/templates.yaml b/ccs/promptsource/templates/sem_eval_2014_task_1/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/sem_eval_2014_task_1/templates.yaml
rename to ccs/promptsource/templates/sem_eval_2014_task_1/templates.yaml
diff --git a/elk/promptsource/templates/sent_comp/templates.yaml b/ccs/promptsource/templates/sent_comp/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/sent_comp/templates.yaml
rename to ccs/promptsource/templates/sent_comp/templates.yaml
diff --git a/elk/promptsource/templates/sick/templates.yaml b/ccs/promptsource/templates/sick/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/sick/templates.yaml
rename to ccs/promptsource/templates/sick/templates.yaml
diff --git a/elk/promptsource/templates/sms_spam/templates.yaml b/ccs/promptsource/templates/sms_spam/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/sms_spam/templates.yaml
rename to ccs/promptsource/templates/sms_spam/templates.yaml
diff --git a/elk/promptsource/templates/snips_built_in_intents/templates.yaml b/ccs/promptsource/templates/snips_built_in_intents/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/snips_built_in_intents/templates.yaml
rename to ccs/promptsource/templates/snips_built_in_intents/templates.yaml
diff --git a/elk/promptsource/templates/snli/templates.yaml b/ccs/promptsource/templates/snli/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/snli/templates.yaml
rename to ccs/promptsource/templates/snli/templates.yaml
diff --git a/elk/promptsource/templates/social_i_qa/templates.yaml b/ccs/promptsource/templates/social_i_qa/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/social_i_qa/templates.yaml
rename to ccs/promptsource/templates/social_i_qa/templates.yaml
diff --git a/elk/promptsource/templates/species_800/templates.yaml b/ccs/promptsource/templates/species_800/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/species_800/templates.yaml
rename to ccs/promptsource/templates/species_800/templates.yaml
diff --git a/elk/promptsource/templates/squad/templates.yaml b/ccs/promptsource/templates/squad/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/squad/templates.yaml
rename to ccs/promptsource/templates/squad/templates.yaml
diff --git a/elk/promptsource/templates/squad_adversarial/AddSent/templates.yaml b/ccs/promptsource/templates/squad_adversarial/AddSent/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/squad_adversarial/AddSent/templates.yaml
rename to ccs/promptsource/templates/squad_adversarial/AddSent/templates.yaml
diff --git a/elk/promptsource/templates/squad_v2/templates.yaml b/ccs/promptsource/templates/squad_v2/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/squad_v2/templates.yaml
rename to ccs/promptsource/templates/squad_v2/templates.yaml
diff --git a/elk/promptsource/templates/squadshifts/amazon/templates.yaml b/ccs/promptsource/templates/squadshifts/amazon/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/squadshifts/amazon/templates.yaml
rename to ccs/promptsource/templates/squadshifts/amazon/templates.yaml
diff --git a/elk/promptsource/templates/squadshifts/new_wiki/templates.yaml b/ccs/promptsource/templates/squadshifts/new_wiki/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/squadshifts/new_wiki/templates.yaml
rename to ccs/promptsource/templates/squadshifts/new_wiki/templates.yaml
diff --git a/elk/promptsource/templates/squadshifts/nyt/templates.yaml b/ccs/promptsource/templates/squadshifts/nyt/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/squadshifts/nyt/templates.yaml
rename to ccs/promptsource/templates/squadshifts/nyt/templates.yaml
diff --git a/elk/promptsource/templates/sst/default/templates.yaml b/ccs/promptsource/templates/sst/default/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/sst/default/templates.yaml
rename to ccs/promptsource/templates/sst/default/templates.yaml
diff --git a/elk/promptsource/templates/sst2/templates.yaml b/ccs/promptsource/templates/sst2/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/sst2/templates.yaml
rename to ccs/promptsource/templates/sst2/templates.yaml
diff --git a/elk/promptsource/templates/story_cloze/2016/templates.yaml b/ccs/promptsource/templates/story_cloze/2016/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/story_cloze/2016/templates.yaml
rename to ccs/promptsource/templates/story_cloze/2016/templates.yaml
diff --git a/elk/promptsource/templates/stsb_multi_mt/en/templates.yaml b/ccs/promptsource/templates/stsb_multi_mt/en/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/stsb_multi_mt/en/templates.yaml
rename to ccs/promptsource/templates/stsb_multi_mt/en/templates.yaml
diff --git a/elk/promptsource/templates/subjqa/books/templates.yaml b/ccs/promptsource/templates/subjqa/books/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/subjqa/books/templates.yaml
rename to ccs/promptsource/templates/subjqa/books/templates.yaml
diff --git a/elk/promptsource/templates/subjqa/electronics/templates.yaml b/ccs/promptsource/templates/subjqa/electronics/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/subjqa/electronics/templates.yaml
rename to ccs/promptsource/templates/subjqa/electronics/templates.yaml
diff --git a/elk/promptsource/templates/subjqa/grocery/templates.yaml b/ccs/promptsource/templates/subjqa/grocery/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/subjqa/grocery/templates.yaml
rename to ccs/promptsource/templates/subjqa/grocery/templates.yaml
diff --git a/elk/promptsource/templates/subjqa/movies/templates.yaml b/ccs/promptsource/templates/subjqa/movies/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/subjqa/movies/templates.yaml
rename to ccs/promptsource/templates/subjqa/movies/templates.yaml
diff --git a/elk/promptsource/templates/subjqa/restaurants/templates.yaml b/ccs/promptsource/templates/subjqa/restaurants/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/subjqa/restaurants/templates.yaml
rename to ccs/promptsource/templates/subjqa/restaurants/templates.yaml
diff --git a/elk/promptsource/templates/subjqa/tripadvisor/templates.yaml b/ccs/promptsource/templates/subjqa/tripadvisor/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/subjqa/tripadvisor/templates.yaml
rename to ccs/promptsource/templates/subjqa/tripadvisor/templates.yaml
diff --git a/elk/promptsource/templates/super_glue/axb/templates.yaml b/ccs/promptsource/templates/super_glue/axb/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/super_glue/axb/templates.yaml
rename to ccs/promptsource/templates/super_glue/axb/templates.yaml
diff --git a/elk/promptsource/templates/super_glue/axg/templates.yaml b/ccs/promptsource/templates/super_glue/axg/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/super_glue/axg/templates.yaml
rename to ccs/promptsource/templates/super_glue/axg/templates.yaml
diff --git a/elk/promptsource/templates/super_glue/boolq/templates.yaml b/ccs/promptsource/templates/super_glue/boolq/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/super_glue/boolq/templates.yaml
rename to ccs/promptsource/templates/super_glue/boolq/templates.yaml
diff --git a/elk/promptsource/templates/super_glue/cb/templates.yaml b/ccs/promptsource/templates/super_glue/cb/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/super_glue/cb/templates.yaml
rename to ccs/promptsource/templates/super_glue/cb/templates.yaml
diff --git a/elk/promptsource/templates/super_glue/copa/templates.yaml b/ccs/promptsource/templates/super_glue/copa/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/super_glue/copa/templates.yaml
rename to ccs/promptsource/templates/super_glue/copa/templates.yaml
diff --git a/elk/promptsource/templates/super_glue/multirc/templates.yaml b/ccs/promptsource/templates/super_glue/multirc/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/super_glue/multirc/templates.yaml
rename to ccs/promptsource/templates/super_glue/multirc/templates.yaml
diff --git a/elk/promptsource/templates/super_glue/record/templates.yaml b/ccs/promptsource/templates/super_glue/record/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/super_glue/record/templates.yaml
rename to ccs/promptsource/templates/super_glue/record/templates.yaml
diff --git a/elk/promptsource/templates/super_glue/rte/templates.yaml b/ccs/promptsource/templates/super_glue/rte/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/super_glue/rte/templates.yaml
rename to ccs/promptsource/templates/super_glue/rte/templates.yaml
diff --git a/elk/promptsource/templates/super_glue/wic/templates.yaml b/ccs/promptsource/templates/super_glue/wic/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/super_glue/wic/templates.yaml
rename to ccs/promptsource/templates/super_glue/wic/templates.yaml
diff --git a/elk/promptsource/templates/super_glue/wsc.fixed/templates.yaml b/ccs/promptsource/templates/super_glue/wsc.fixed/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/super_glue/wsc.fixed/templates.yaml
rename to ccs/promptsource/templates/super_glue/wsc.fixed/templates.yaml
diff --git a/elk/promptsource/templates/swag/regular/templates.yaml b/ccs/promptsource/templates/swag/regular/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/swag/regular/templates.yaml
rename to ccs/promptsource/templates/swag/regular/templates.yaml
diff --git a/elk/promptsource/templates/tab_fact/tab_fact/templates.yaml b/ccs/promptsource/templates/tab_fact/tab_fact/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/tab_fact/tab_fact/templates.yaml
rename to ccs/promptsource/templates/tab_fact/tab_fact/templates.yaml
diff --git a/elk/promptsource/templates/tmu_gfm_dataset/templates.yaml b/ccs/promptsource/templates/tmu_gfm_dataset/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/tmu_gfm_dataset/templates.yaml
rename to ccs/promptsource/templates/tmu_gfm_dataset/templates.yaml
diff --git a/elk/promptsource/templates/trec/templates.yaml b/ccs/promptsource/templates/trec/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/trec/templates.yaml
rename to ccs/promptsource/templates/trec/templates.yaml
diff --git a/elk/promptsource/templates/trivia_qa/unfiltered/templates.yaml b/ccs/promptsource/templates/trivia_qa/unfiltered/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/trivia_qa/unfiltered/templates.yaml
rename to ccs/promptsource/templates/trivia_qa/unfiltered/templates.yaml
diff --git a/elk/promptsource/templates/turk/templates.yaml b/ccs/promptsource/templates/turk/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/turk/templates.yaml
rename to ccs/promptsource/templates/turk/templates.yaml
diff --git a/elk/promptsource/templates/tweet_eval/emoji/templates.yaml b/ccs/promptsource/templates/tweet_eval/emoji/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/tweet_eval/emoji/templates.yaml
rename to ccs/promptsource/templates/tweet_eval/emoji/templates.yaml
diff --git a/elk/promptsource/templates/tweet_eval/emotion/templates.yaml b/ccs/promptsource/templates/tweet_eval/emotion/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/tweet_eval/emotion/templates.yaml
rename to ccs/promptsource/templates/tweet_eval/emotion/templates.yaml
diff --git a/elk/promptsource/templates/tweet_eval/hate/templates.yaml b/ccs/promptsource/templates/tweet_eval/hate/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/tweet_eval/hate/templates.yaml
rename to ccs/promptsource/templates/tweet_eval/hate/templates.yaml
diff --git a/elk/promptsource/templates/tweet_eval/irony/templates.yaml b/ccs/promptsource/templates/tweet_eval/irony/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/tweet_eval/irony/templates.yaml
rename to ccs/promptsource/templates/tweet_eval/irony/templates.yaml
diff --git a/elk/promptsource/templates/tweet_eval/offensive/templates.yaml b/ccs/promptsource/templates/tweet_eval/offensive/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/tweet_eval/offensive/templates.yaml
rename to ccs/promptsource/templates/tweet_eval/offensive/templates.yaml
diff --git a/elk/promptsource/templates/tweet_eval/sentiment/templates.yaml b/ccs/promptsource/templates/tweet_eval/sentiment/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/tweet_eval/sentiment/templates.yaml
rename to ccs/promptsource/templates/tweet_eval/sentiment/templates.yaml
diff --git a/elk/promptsource/templates/tweet_eval/stance_abortion/templates.yaml b/ccs/promptsource/templates/tweet_eval/stance_abortion/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/tweet_eval/stance_abortion/templates.yaml
rename to ccs/promptsource/templates/tweet_eval/stance_abortion/templates.yaml
diff --git a/elk/promptsource/templates/tweet_eval/stance_atheism/templates.yaml b/ccs/promptsource/templates/tweet_eval/stance_atheism/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/tweet_eval/stance_atheism/templates.yaml
rename to ccs/promptsource/templates/tweet_eval/stance_atheism/templates.yaml
diff --git a/elk/promptsource/templates/tweet_eval/stance_climate/templates.yaml b/ccs/promptsource/templates/tweet_eval/stance_climate/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/tweet_eval/stance_climate/templates.yaml
rename to ccs/promptsource/templates/tweet_eval/stance_climate/templates.yaml
diff --git a/elk/promptsource/templates/tweet_eval/stance_feminist/templates.yaml b/ccs/promptsource/templates/tweet_eval/stance_feminist/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/tweet_eval/stance_feminist/templates.yaml
rename to ccs/promptsource/templates/tweet_eval/stance_feminist/templates.yaml
diff --git a/elk/promptsource/templates/tweet_eval/stance_hillary/templates.yaml b/ccs/promptsource/templates/tweet_eval/stance_hillary/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/tweet_eval/stance_hillary/templates.yaml
rename to ccs/promptsource/templates/tweet_eval/stance_hillary/templates.yaml
diff --git a/elk/promptsource/templates/tydiqa/primary_task/templates.yaml b/ccs/promptsource/templates/tydiqa/primary_task/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/tydiqa/primary_task/templates.yaml
rename to ccs/promptsource/templates/tydiqa/primary_task/templates.yaml
diff --git a/elk/promptsource/templates/tydiqa/secondary_task/templates.yaml b/ccs/promptsource/templates/tydiqa/secondary_task/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/tydiqa/secondary_task/templates.yaml
rename to ccs/promptsource/templates/tydiqa/secondary_task/templates.yaml
diff --git a/elk/promptsource/templates/web_questions/templates.yaml b/ccs/promptsource/templates/web_questions/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/web_questions/templates.yaml
rename to ccs/promptsource/templates/web_questions/templates.yaml
diff --git a/elk/promptsource/templates/wiki_bio/templates.yaml b/ccs/promptsource/templates/wiki_bio/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/wiki_bio/templates.yaml
rename to ccs/promptsource/templates/wiki_bio/templates.yaml
diff --git a/elk/promptsource/templates/wiki_hop/masked/templates.yaml b/ccs/promptsource/templates/wiki_hop/masked/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/wiki_hop/masked/templates.yaml
rename to ccs/promptsource/templates/wiki_hop/masked/templates.yaml
diff --git a/elk/promptsource/templates/wiki_hop/original/templates.yaml b/ccs/promptsource/templates/wiki_hop/original/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/wiki_hop/original/templates.yaml
rename to ccs/promptsource/templates/wiki_hop/original/templates.yaml
diff --git a/elk/promptsource/templates/wiki_qa/templates.yaml b/ccs/promptsource/templates/wiki_qa/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/wiki_qa/templates.yaml
rename to ccs/promptsource/templates/wiki_qa/templates.yaml
diff --git a/elk/promptsource/templates/wiki_split/templates.yaml b/ccs/promptsource/templates/wiki_split/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/wiki_split/templates.yaml
rename to ccs/promptsource/templates/wiki_split/templates.yaml
diff --git a/elk/promptsource/templates/wino_bias/type1_anti/templates.yaml b/ccs/promptsource/templates/wino_bias/type1_anti/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/wino_bias/type1_anti/templates.yaml
rename to ccs/promptsource/templates/wino_bias/type1_anti/templates.yaml
diff --git a/elk/promptsource/templates/wino_bias/type1_pro/templates.yaml b/ccs/promptsource/templates/wino_bias/type1_pro/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/wino_bias/type1_pro/templates.yaml
rename to ccs/promptsource/templates/wino_bias/type1_pro/templates.yaml
diff --git a/elk/promptsource/templates/wino_bias/type2_anti/templates.yaml b/ccs/promptsource/templates/wino_bias/type2_anti/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/wino_bias/type2_anti/templates.yaml
rename to ccs/promptsource/templates/wino_bias/type2_anti/templates.yaml
diff --git a/elk/promptsource/templates/wino_bias/type2_pro/templates.yaml b/ccs/promptsource/templates/wino_bias/type2_pro/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/wino_bias/type2_pro/templates.yaml
rename to ccs/promptsource/templates/wino_bias/type2_pro/templates.yaml
diff --git a/elk/promptsource/templates/winograd_wsc/wsc273/templates.yaml b/ccs/promptsource/templates/winograd_wsc/wsc273/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/winograd_wsc/wsc273/templates.yaml
rename to ccs/promptsource/templates/winograd_wsc/wsc273/templates.yaml
diff --git a/elk/promptsource/templates/winograd_wsc/wsc285/templates.yaml b/ccs/promptsource/templates/winograd_wsc/wsc285/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/winograd_wsc/wsc285/templates.yaml
rename to ccs/promptsource/templates/winograd_wsc/wsc285/templates.yaml
diff --git a/elk/promptsource/templates/winogrande/winogrande_debiased/templates.yaml b/ccs/promptsource/templates/winogrande/winogrande_debiased/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/winogrande/winogrande_debiased/templates.yaml
rename to ccs/promptsource/templates/winogrande/winogrande_debiased/templates.yaml
diff --git a/elk/promptsource/templates/winogrande/winogrande_l/templates.yaml b/ccs/promptsource/templates/winogrande/winogrande_l/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/winogrande/winogrande_l/templates.yaml
rename to ccs/promptsource/templates/winogrande/winogrande_l/templates.yaml
diff --git a/elk/promptsource/templates/winogrande/winogrande_m/templates.yaml b/ccs/promptsource/templates/winogrande/winogrande_m/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/winogrande/winogrande_m/templates.yaml
rename to ccs/promptsource/templates/winogrande/winogrande_m/templates.yaml
diff --git a/elk/promptsource/templates/winogrande/winogrande_s/templates.yaml b/ccs/promptsource/templates/winogrande/winogrande_s/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/winogrande/winogrande_s/templates.yaml
rename to ccs/promptsource/templates/winogrande/winogrande_s/templates.yaml
diff --git a/elk/promptsource/templates/winogrande/winogrande_xl/templates.yaml b/ccs/promptsource/templates/winogrande/winogrande_xl/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/winogrande/winogrande_xl/templates.yaml
rename to ccs/promptsource/templates/winogrande/winogrande_xl/templates.yaml
diff --git a/elk/promptsource/templates/winogrande/winogrande_xs/templates.yaml b/ccs/promptsource/templates/winogrande/winogrande_xs/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/winogrande/winogrande_xs/templates.yaml
rename to ccs/promptsource/templates/winogrande/winogrande_xs/templates.yaml
diff --git a/elk/promptsource/templates/wiqa/templates.yaml b/ccs/promptsource/templates/wiqa/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/wiqa/templates.yaml
rename to ccs/promptsource/templates/wiqa/templates.yaml
diff --git a/elk/promptsource/templates/xnli/en/templates.yaml b/ccs/promptsource/templates/xnli/en/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/xnli/en/templates.yaml
rename to ccs/promptsource/templates/xnli/en/templates.yaml
diff --git a/elk/promptsource/templates/xquad/xquad.en/templates.yaml b/ccs/promptsource/templates/xquad/xquad.en/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/xquad/xquad.en/templates.yaml
rename to ccs/promptsource/templates/xquad/xquad.en/templates.yaml
diff --git a/elk/promptsource/templates/xquad_r/en/templates.yaml b/ccs/promptsource/templates/xquad_r/en/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/xquad_r/en/templates.yaml
rename to ccs/promptsource/templates/xquad_r/en/templates.yaml
diff --git a/elk/promptsource/templates/xsum/templates.yaml b/ccs/promptsource/templates/xsum/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/xsum/templates.yaml
rename to ccs/promptsource/templates/xsum/templates.yaml
diff --git a/elk/promptsource/templates/yahoo_answers_qa/templates.yaml b/ccs/promptsource/templates/yahoo_answers_qa/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/yahoo_answers_qa/templates.yaml
rename to ccs/promptsource/templates/yahoo_answers_qa/templates.yaml
diff --git a/elk/promptsource/templates/yahoo_answers_topics/templates.yaml b/ccs/promptsource/templates/yahoo_answers_topics/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/yahoo_answers_topics/templates.yaml
rename to ccs/promptsource/templates/yahoo_answers_topics/templates.yaml
diff --git a/elk/promptsource/templates/yelp_polarity/templates.yaml b/ccs/promptsource/templates/yelp_polarity/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/yelp_polarity/templates.yaml
rename to ccs/promptsource/templates/yelp_polarity/templates.yaml
diff --git a/elk/promptsource/templates/yelp_review_full/templates.yaml b/ccs/promptsource/templates/yelp_review_full/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/yelp_review_full/templates.yaml
rename to ccs/promptsource/templates/yelp_review_full/templates.yaml
diff --git a/elk/promptsource/templates/zest/templates.yaml b/ccs/promptsource/templates/zest/templates.yaml
similarity index 100%
rename from elk/promptsource/templates/zest/templates.yaml
rename to ccs/promptsource/templates/zest/templates.yaml
diff --git a/elk/resources/adjectives.json b/ccs/resources/adjectives.json
similarity index 100%
rename from elk/resources/adjectives.json
rename to ccs/resources/adjectives.json
diff --git a/elk/resources/names.json b/ccs/resources/names.json
similarity index 100%
rename from elk/resources/names.json
rename to ccs/resources/names.json
diff --git a/elk/run.py b/ccs/run.py
similarity index 97%
rename from elk/run.py
rename to ccs/run.py
index fb8903cc..efa8a392 100644
--- a/elk/run.py
+++ b/ccs/run.py
@@ -20,7 +20,7 @@
 from .debug_logging import save_debug_log
 from .extraction import Extract, extract
 from .extraction.dataset_name import DatasetDictWithName
-from .files import elk_reporter_dir, memorably_named_dir
+from .files import ccs_reporter_dir, memorably_named_dir
 from .utils import (
     Color,
     assert_type,
@@ -72,9 +72,9 @@ def execute(
 
         if self.out_dir is None:
             # Save in a memorably-named directory inside of
-            # ELK_REPORTER_DIR/<model_name>/<dataset_name>
+            # CCS_REPORTER_DIR/<model_name>/<dataset_name>
             ds_name = "+".join(self.data.datasets)
-            root = elk_reporter_dir() / self.data.model / ds_name
+            root = ccs_reporter_dir() / self.data.model / ds_name
 
             self.out_dir = memorably_named_dir(root)
 
diff --git a/elk/training/__init__.py b/ccs/training/__init__.py
similarity index 100%
rename from elk/training/__init__.py
rename to ccs/training/__init__.py
diff --git a/elk/training/burns_norm.py b/ccs/training/burns_norm.py
similarity index 100%
rename from elk/training/burns_norm.py
rename to ccs/training/burns_norm.py
diff --git a/elk/training/ccs_reporter.py b/ccs/training/ccs_reporter.py
similarity index 99%
rename from elk/training/ccs_reporter.py
rename to ccs/training/ccs_reporter.py
index 7a55a858..ce20fee3 100644
--- a/elk/training/ccs_reporter.py
+++ b/ccs/training/ccs_reporter.py
@@ -38,7 +38,7 @@ class CcsConfig(FitterConfig):
     loss: list[str] = field(default_factory=lambda: ["ccs"])
     """
     The loss function to use. list of strings, each of the form "coef*name", where coef
-    is a float and name is one of the keys in `elk.training.losses.LOSSES`.
+    is a float and name is one of the keys in `ccs.training.losses.LOSSES`.
     Example: `--loss 1.0*consistency_squared 0.5*prompt_var` corresponds to the loss
     function 1.0*consistency_squared + 0.5*prompt_var.
     """
diff --git a/elk/training/classifier.py b/ccs/training/classifier.py
similarity index 100%
rename from elk/training/classifier.py
rename to ccs/training/classifier.py
diff --git a/elk/training/common.py b/ccs/training/common.py
similarity index 100%
rename from elk/training/common.py
rename to ccs/training/common.py
diff --git a/elk/training/eigen_reporter.py b/ccs/training/eigen_reporter.py
similarity index 100%
rename from elk/training/eigen_reporter.py
rename to ccs/training/eigen_reporter.py
diff --git a/elk/training/losses.py b/ccs/training/losses.py
similarity index 100%
rename from elk/training/losses.py
rename to ccs/training/losses.py
diff --git a/elk/training/platt_scaling.py b/ccs/training/platt_scaling.py
similarity index 100%
rename from elk/training/platt_scaling.py
rename to ccs/training/platt_scaling.py
diff --git a/elk/training/supervised.py b/ccs/training/supervised.py
similarity index 100%
rename from elk/training/supervised.py
rename to ccs/training/supervised.py
diff --git a/elk/training/sweep.py b/ccs/training/sweep.py
similarity index 100%
rename from elk/training/sweep.py
rename to ccs/training/sweep.py
diff --git a/elk/training/train.py b/ccs/training/train.py
similarity index 100%
rename from elk/training/train.py
rename to ccs/training/train.py
diff --git a/elk/truncated_eigh.py b/ccs/truncated_eigh.py
similarity index 100%
rename from elk/truncated_eigh.py
rename to ccs/truncated_eigh.py
diff --git a/elk/utils/__init__.py b/ccs/utils/__init__.py
similarity index 100%
rename from elk/utils/__init__.py
rename to ccs/utils/__init__.py
diff --git a/elk/utils/constants.py b/ccs/utils/constants.py
similarity index 100%
rename from elk/utils/constants.py
rename to ccs/utils/constants.py
diff --git a/elk/utils/data_utils.py b/ccs/utils/data_utils.py
similarity index 100%
rename from elk/utils/data_utils.py
rename to ccs/utils/data_utils.py
diff --git a/elk/utils/gpu_utils.py b/ccs/utils/gpu_utils.py
similarity index 100%
rename from elk/utils/gpu_utils.py
rename to ccs/utils/gpu_utils.py
diff --git a/elk/utils/hf_utils.py b/ccs/utils/hf_utils.py
similarity index 100%
rename from elk/utils/hf_utils.py
rename to ccs/utils/hf_utils.py
diff --git a/elk/utils/math_util.py b/ccs/utils/math_util.py
similarity index 100%
rename from elk/utils/math_util.py
rename to ccs/utils/math_util.py
diff --git a/elk/utils/pretty.py b/ccs/utils/pretty.py
similarity index 100%
rename from elk/utils/pretty.py
rename to ccs/utils/pretty.py
diff --git a/elk/utils/tree_utils.py b/ccs/utils/tree_utils.py
similarity index 100%
rename from elk/utils/tree_utils.py
rename to ccs/utils/tree_utils.py
diff --git a/elk/utils/typing.py b/ccs/utils/typing.py
similarity index 100%
rename from elk/utils/typing.py
rename to ccs/utils/typing.py
diff --git a/pyproject.toml b/pyproject.toml
index cb004269..2c1becb3 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -3,8 +3,8 @@ requires = ["setuptools"]
 build-backend = "setuptools.build_meta"
 
 [project]
-name = "eleuther-elk"
-description = "Keeping language models honest by directly eliciting knowledge encoded in their activations"
+name = "eleuther-ccs"
+description = "Unsupervised methods of eliciting latent knowledge related to Contrast Consistent Search (CCS)"
 readme = "README.md"
 requires-python = ">=3.10"
 keywords = ["nlp", "interpretability", "language-models", "explainable-ai"]
@@ -58,20 +58,20 @@ dev = [
 ]
 
 [project.scripts]
-elk = "elk.__main__:run"
+ccs = "ccs.__main__:run"
 
 [tool.pyright]
-include = ["elk*"]
+include = ["ccs*"]
 reportPrivateImportUsage = false
 
 [tool.pytest.ini_options]
 testpaths = ["tests"]
 
 [tool.setuptools.package-data]
-elk = ["*.json", "*.yaml"]
+ccs = ["*.json", "*.yaml"]
 
 [tool.setuptools.packages.find]
-include = ["elk"]
+include = ["ccs"]
 
 [tool.ruff]
 # Enable pycodestyle (`E`), Pyflakes (`F`), and isort (`I`) codes
diff --git a/tests/test_burns_norm.py b/tests/test_burns_norm.py
index 4620ae0e..09f8fb4a 100644
--- a/tests/test_burns_norm.py
+++ b/tests/test_burns_norm.py
@@ -1,7 +1,7 @@
 import torch
 from torch import Tensor
 
-from elk.training.burns_norm import BurnsNorm
+from ccs.training.burns_norm import BurnsNorm
 
 
 def correct_but_slow_normalization(x_all: Tensor, scale=True) -> Tensor:
diff --git a/tests/test_classifier.py b/tests/test_classifier.py
index bdc9023d..444c0f0c 100644
--- a/tests/test_classifier.py
+++ b/tests/test_classifier.py
@@ -2,7 +2,7 @@
 from sklearn.datasets import make_classification
 from sklearn.linear_model import LogisticRegression
 
-from elk.training.classifier import Classifier
+from ccs.training.classifier import Classifier
 
 
 @torch.no_grad()
diff --git a/tests/test_eigen_reporter.py b/tests/test_eigen_reporter.py
index 6303cb03..0edefc77 100644
--- a/tests/test_eigen_reporter.py
+++ b/tests/test_eigen_reporter.py
@@ -1,7 +1,7 @@
 import torch
 
-from elk.training import EigenFitter, EigenFitterConfig
-from elk.utils import batch_cov, cov_mean_fused
+from ccs.training import EigenFitter, EigenFitterConfig
+from ccs.utils import batch_cov, cov_mean_fused
 
 
 def test_eigen_reporter():
diff --git a/tests/test_load_prompts.py b/tests/test_load_prompts.py
index 0d309cf5..041c1893 100644
--- a/tests/test_load_prompts.py
+++ b/tests/test_load_prompts.py
@@ -3,8 +3,8 @@
 
 import pytest
 
-from elk.extraction import Extract, load_prompts
-from elk.promptsource.templates import DatasetTemplates
+from ccs.extraction import Extract, load_prompts
+from ccs.promptsource.templates import DatasetTemplates
 
 
 @pytest.mark.filterwarnings("ignore:Unable to find a decoding function")
diff --git a/tests/test_math.py b/tests/test_math.py
index 34984d8f..29289d34 100644
--- a/tests/test_math.py
+++ b/tests/test_math.py
@@ -6,7 +6,7 @@
 from hypothesis import given
 from hypothesis import strategies as st
 
-from elk.utils import batch_cov, cov_mean_fused, stochastic_round_constrained
+from ccs.utils import batch_cov, cov_mean_fused, stochastic_round_constrained
 
 
 def test_cov_mean_fused():
diff --git a/tests/test_metrics.py b/tests/test_metrics.py
index 3510a302..94b17c0f 100644
--- a/tests/test_metrics.py
+++ b/tests/test_metrics.py
@@ -7,7 +7,7 @@
 from sklearn.metrics import roc_auc_score
 from torch.distributions.normal import Normal
 
-from elk.metrics import accuracy_ci, roc_auc
+from ccs.metrics import accuracy_ci, roc_auc
 
 
 def test_auroc_and_acc():
diff --git a/tests/test_samplers.py b/tests/test_samplers.py
index 9a1f694e..0857e60d 100644
--- a/tests/test_samplers.py
+++ b/tests/test_samplers.py
@@ -4,8 +4,8 @@
 
 from datasets import IterableDataset, load_dataset
 
-from elk.extraction import BalancedSampler, FewShotSampler
-from elk.utils import assert_type, infer_label_column
+from ccs.extraction import BalancedSampler, FewShotSampler
+from ccs.utils import assert_type, infer_label_column
 
 
 def test_output_batches_are_balanced():
diff --git a/tests/test_smoke_elicit.py b/tests/test_smoke_elicit.py
index bac0f398..0b8b97a0 100644
--- a/tests/test_smoke_elicit.py
+++ b/tests/test_smoke_elicit.py
@@ -1,8 +1,8 @@
 from pathlib import Path
 
-from elk import Extract
-from elk.training import CcsConfig, EigenFitterConfig
-from elk.training.train import Elicit
+from ccs import Extract
+from ccs.training import CcsConfig, EigenFitterConfig
+from ccs.training.train import Elicit
 
 
 def test_smoke_elicit_run_tiny_gpt2_ccs(tmp_path: Path):
diff --git a/tests/test_smoke_eval.py b/tests/test_smoke_eval.py
index 4efd7112..19e1257c 100644
--- a/tests/test_smoke_eval.py
+++ b/tests/test_smoke_eval.py
@@ -2,10 +2,10 @@
 
 import pandas as pd
 
-from elk import Extract
-from elk.evaluation import Eval
-from elk.training import CcsConfig, EigenFitterConfig
-from elk.training.train import Elicit
+from ccs import Extract
+from ccs.evaluation import Eval
+from ccs.training import CcsConfig, EigenFitterConfig
+from ccs.training.train import Elicit
 
 EVAL_EXPECTED_FILES = [
     "cfg.yaml",
diff --git a/tests/test_truncated_eigh.py b/tests/test_truncated_eigh.py
index 5241f1c0..63d06722 100644
--- a/tests/test_truncated_eigh.py
+++ b/tests/test_truncated_eigh.py
@@ -3,7 +3,7 @@
 import torch
 from scipy.sparse.linalg import eigsh
 
-from elk.truncated_eigh import truncated_eigh
+from ccs.truncated_eigh import truncated_eigh
 
 
 def random_symmetric_matrix(n: int, k: int) -> torch.Tensor:
diff --git a/tests/test_viz.py b/tests/test_viz.py
deleted file mode 100644
index fe3214b0..00000000
--- a/tests/test_viz.py
+++ /dev/null
@@ -1,31 +0,0 @@
-from pathlib import Path
-
-import pytest
-
-from elk.plotting.visualize import SweepVisualization
-
-
-@pytest.fixture
-def setup_fs(fs):
-    test_dir = "/sweep1"
-    fs.create_dir(test_dir)
-    fs.create_dir(f"{test_dir}/huggyllama/llama-13b/imdb")
-    fs.create_file(f"{test_dir}/huggyllama/llama-13b/imdb/eval.csv")
-    fs.create_dir(f"{test_dir}/huggyllama/llama-12b/news")
-    fs.create_file(f"{test_dir}/huggyllama/llama-12b/news/eval.csv")
-    fs.create_file(f"{test_dir}/gpt2-medium/imdb/eval.csv")
-
-    return Path(test_dir)
-
-
-def test_get_model_paths(setup_fs):
-    test_dir = setup_fs
-    result = SweepVisualization._get_model_paths(test_dir)
-
-    root = Path(test_dir)
-    for path in root.rglob("*"):
-        print(path)
-    assert len(result) == 3
-    assert any([p.name == "llama-13b" for p in result])
-    assert any([p.name == "llama-12b" for p in result])
-    assert any([p.name == "gpt2-medium" for p in result])

From 2deefc30ecf96672e3349da0280bc807401de58b Mon Sep 17 00:00:00 2001
From: Alex Mallen <atmallen8@gmail.com>
Date: Fri, 10 Nov 2023 18:26:30 +0000
Subject: [PATCH 62/77] fix type issue

---
 ccs/metrics/eval.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ccs/metrics/eval.py b/ccs/metrics/eval.py
index d0e2bf7a..6bd31d1e 100644
--- a/ccs/metrics/eval.py
+++ b/ccs/metrics/eval.py
@@ -1,5 +1,5 @@
 from dataclasses import asdict, dataclass
-from typing import Literal
+from typing import Any, Literal
 
 import torch
 from einops import repeat
@@ -26,7 +26,7 @@ class EvalResult:
     cal_thresh: float | None
     """The threshold used to compute the calibrated accuracy."""
 
-    def to_dict(self, prefix: str = "") -> dict[str, float]:
+    def to_dict(self, prefix: str = "") -> dict[str, Any]:
         """Convert the result to a dictionary."""
         acc_dict = {f"{prefix}acc_{k}": v for k, v in asdict(self.accuracy).items()}
         cal_acc_dict = (

From 100635ce93c896aab50cc133370b938d85552ebb Mon Sep 17 00:00:00 2001
From: Alex Mallen <atmallen8@gmail.com>
Date: Fri, 10 Nov 2023 22:43:29 +0000
Subject: [PATCH 63/77] save logprobs

---
 ccs/debug_logging.py             |   6 +-
 ccs/evaluation/evaluate.py       |  51 ++++++++--
 ccs/extraction/extraction.py     |  24 +++--
 ccs/extraction/prompt_loading.py |  17 +++-
 ccs/metrics/__init__.py          |   3 +-
 ccs/metrics/eval.py              |  24 +++++
 ccs/promptsource/templates.py    |   2 +-
 ccs/run.py                       |  88 ++++++++++++++---
 ccs/training/classifier.py       |   2 +-
 ccs/training/supervised.py       |  11 ++-
 ccs/training/train.py            | 165 ++++++++++++++++++++-----------
 11 files changed, 298 insertions(+), 95 deletions(-)

diff --git a/ccs/debug_logging.py b/ccs/debug_logging.py
index 59bea62f..fe6e7f02 100644
--- a/ccs/debug_logging.py
+++ b/ccs/debug_logging.py
@@ -31,7 +31,11 @@ def save_debug_log(datasets: list[DatasetDictWithName], out_dir: Path) -> None:
         else:
             train_split, val_split = select_train_val_splits(ds)
 
-        text_questions = ds[val_split][0]["text_questions"]
+        if len(ds[val_split]) == 0:
+            logging.warning(f"Val split '{val_split}' is empty!")
+            continue
+
+        text_questions = ds[val_split][0]["texts"]
         template_ids = ds[val_split][0]["variant_ids"]
         label = ds[val_split][0]["label"]
 
diff --git a/ccs/evaluation/evaluate.py b/ccs/evaluation/evaluate.py
index 53df3715..46ddf6e6 100644
--- a/ccs/evaluation/evaluate.py
+++ b/ccs/evaluation/evaluate.py
@@ -7,7 +7,7 @@
 from simple_parsing.helpers import field
 
 from ..files import ccs_reporter_dir
-from ..metrics import evaluate_preds
+from ..metrics import evaluate_preds, get_logprobs
 from ..run import Run
 from ..utils import Color
 
@@ -31,7 +31,7 @@ def execute(self, highlight_color: Color = "cyan"):
     @torch.inference_mode()
     def apply_to_layer(
         self, layer: int, devices: list[str], world_size: int
-    ) -> dict[str, pd.DataFrame]:
+    ) -> tuple[dict[str, pd.DataFrame], dict]:
         """Evaluate a single reporter on a single layer."""
         device = self.get_device(devices, world_size)
         val_output = self.prepare_data(device, layer, "val")
@@ -41,28 +41,51 @@ def apply_to_layer(
         reporter_path = experiment_dir / "reporters" / f"layer_{layer}.pt"
         reporter = torch.load(reporter_path, map_location=device)
 
+        out_logprobs = defaultdict(dict)
         row_bufs = defaultdict(list)
-        for ds_name, (val_h, val_gt, val_lm_preds) in val_output.items():
+        for ds_name, val_data in val_output.items():
             meta = {"dataset": ds_name, "layer": layer}
+            if self.save_logprobs:
+                out_logprobs[ds_name] = dict(
+                    row_ids=val_data.row_ids.cpu(),
+                    variant_ids=val_data.variant_ids,
+                    texts=val_data.texts,
+                    labels=val_data.labels.cpu(),
+                    lm=dict(),
+                    lr=dict(),
+                    reporter=dict(),
+                )
 
-            val_credences = reporter(val_h)
+            val_credences = reporter(val_data.hiddens)
             for mode in ("none", "partial", "full"):
                 row_bufs["eval"].append(
                     {
                         **meta,
                         "ensembling": mode,
-                        **evaluate_preds(val_gt, val_credences, mode).to_dict(),
+                        **evaluate_preds(
+                            val_data.labels, val_credences, mode
+                        ).to_dict(),
                     }
                 )
+                if self.save_logprobs:
+                    out_logprobs[ds_name]["reporter"][mode] = (
+                        get_logprobs(val_credences, mode).detach().cpu()
+                    )
 
-                if val_lm_preds is not None:
+                if val_data.lm_preds is not None:
                     row_bufs["lm_eval"].append(
                         {
                             **meta,
                             "ensembling": mode,
-                            **evaluate_preds(val_gt, val_lm_preds, mode).to_dict(),
+                            **evaluate_preds(
+                                val_data.labels, val_data.lm_preds, mode
+                            ).to_dict(),
                         }
                     )
+                    if self.save_logprobs:
+                        out_logprobs[ds_name]["lm"][mode] = get_logprobs(
+                            val_data.lm_preds, mode
+                        ).cpu()
 
                 lr_dir = experiment_dir / "lr_models"
                 if not self.skip_supervised and lr_dir.exists():
@@ -71,15 +94,25 @@ def apply_to_layer(
                         if not isinstance(lr_models, list):  # backward compatibility
                             lr_models = [lr_models]
 
+                    if self.save_logprobs:
+                        out_logprobs[ds_name]["lr"][mode] = dict()
+
                     for i, model in enumerate(lr_models):
                         model.eval()
+                        val_credences = model(val_data.hiddens)
+                        if self.save_logprobs:
+                            out_logprobs[ds_name]["lr"][mode][i] = get_logprobs(
+                                val_credences, mode
+                            ).cpu()
                         row_bufs["lr_eval"].append(
                             {
                                 "ensembling": mode,
                                 "inlp_iter": i,
                                 **meta,
-                                **evaluate_preds(val_gt, model(val_h), mode).to_dict(),
+                                **evaluate_preds(
+                                    val_data.labels, val_credences, mode
+                                ).to_dict(),
                             }
                         )
 
-        return {k: pd.DataFrame(v) for k, v in row_bufs.items()}
+        return {k: pd.DataFrame(v) for k, v in row_bufs.items()}, out_logprobs
diff --git a/ccs/extraction/extraction.py b/ccs/extraction/extraction.py
index acd87100..f6e0b4b9 100644
--- a/ccs/extraction/extraction.py
+++ b/ccs/extraction/extraction.py
@@ -78,6 +78,9 @@ class Extract(Serializable):
     """The number of prompt templates to use for each example. If -1, all available
     templates are used."""
 
+    balance: bool = True
+    """Whether to balance the number of examples per class."""
+
     layers: tuple[int, ...] = ()
     """Indices of layers to extract hidden states from. We ignore the embedding,
     have only the output of the transformer layers."""
@@ -189,6 +192,7 @@ def extract_hiddens(
         num_shots=cfg.num_shots,
         split_type=split_type,
         template_path=cfg.template_path,
+        balance=cfg.balance,
         rank=rank,
         world_size=world_size,
         seed=cfg.seed,
@@ -229,7 +233,7 @@ def extract_hiddens(
             )
             for layer_idx in layer_indices
         }
-        lm_logits = torch.empty(
+        lm_log_odds = torch.empty(
             num_variants,
             num_choices,
             device=device,
@@ -289,7 +293,12 @@ def extract_hiddens(
 
                 # Compute the log probability of the answer tokens if available
                 if has_lm_preds:
-                    lm_logits[i, j] = -assert_type(Tensor, outputs.loss)
+                    logprob = -assert_type(Tensor, outputs.loss)
+                    # Convert logprob to logodds to be consistent with reporters
+                    # Because we went through logprobs, logodds corresponding to
+                    # probs near 1 will be somewhat imprecise
+                    # log(p/(1-p)) = log(p) - log(1-p) = logp - log(1 - exp(logp))
+                    lm_log_odds[i, j] = logprob - torch.log1p(-logprob.exp())
 
                 hiddens = (
                     outputs.get("decoder_hidden_states") or outputs["hidden_states"]
@@ -323,14 +332,16 @@ def extract_hiddens(
             continue
 
         out_record: dict[str, Any] = dict(
+            row_id=example["row_id"],
             label=example["label"],
             variant_ids=example["template_names"],
-            text_questions=text_questions,
+            texts=text_questions,
             **hidden_dict,
         )
         if has_lm_preds:
-            out_record["model_logits"] = lm_logits.log_softmax(dim=-1)
+            out_record["lm_log_odds"] = lm_log_odds.log_softmax(dim=-1)
 
+        assert out_record["variant_ids"] == sorted(out_record["variant_ids"])
         num_yielded += 1
         yield out_record
 
@@ -375,12 +386,13 @@ def hidden_features(cfg: Extract) -> tuple[DatasetInfo, Features]:
         for layer in layer_indices
     }
     other_cols = {
+        "row_id": Value(dtype="int64"),
         "variant_ids": Sequence(
             Value(dtype="string"),
             length=num_variants,
         ),
         "label": Value(dtype="int64"),
-        "text_questions": Sequence(
+        "texts": Sequence(
             Sequence(
                 Value(dtype="string"),
             ),
@@ -390,7 +402,7 @@ def hidden_features(cfg: Extract) -> tuple[DatasetInfo, Features]:
 
     # Only add model_logits if the model is an autoregressive model
     if is_autoregressive(model_cfg, not cfg.use_encoder_states):
-        other_cols["model_logits"] = Array2D(
+        other_cols["lm_log_odds"] = Array2D(
             shape=(num_variants, num_classes),
             dtype="float32",
         )
diff --git a/ccs/extraction/prompt_loading.py b/ccs/extraction/prompt_loading.py
index cb42d233..553a1052 100644
--- a/ccs/extraction/prompt_loading.py
+++ b/ccs/extraction/prompt_loading.py
@@ -21,6 +21,7 @@ def load_prompts(
     seed: int = 42,
     split_type: Literal["train", "val"] = "train",
     template_path: str | None = None,
+    balance: bool = True,
     rank: int = 0,
     world_size: int = 1,
 ) -> Iterator[dict]:
@@ -45,7 +46,14 @@ def load_prompts(
     ds_dict = assert_type(dict, load_dataset(ds_name, config_name or None))
     split_name = select_split(ds_dict, split_type)
 
-    ds = assert_type(Dataset, ds_dict[split_name].shuffle(seed=seed))
+    ds = assert_type(Dataset, ds_dict[split_name])
+
+    if "row_id" not in ds.column_names:
+        ds = ds.add_column("row_id", range(len(ds)))  # type: ignore
+    else:
+        print("Found `row_id` column, using it as the example id")
+    ds = ds.shuffle(seed=seed)
+
     if world_size > 1:
         ds = ds.shard(world_size, rank)
 
@@ -89,14 +97,14 @@ def load_prompts(
     else:
         fewshot_iter = None
 
-    if label_column in ds.features:
+    if label_column in ds.features and balance:
         ds = BalancedSampler(
             ds.to_iterable_dataset(),
             set(label_choices),
             label_col=label_column,
         )
     else:
-        if rank == 0:
+        if rank == 0 and balance:
             print("No label column found, not balancing")
         ds = ds.to_iterable_dataset()
 
@@ -123,7 +131,7 @@ def _convert_to_prompts(
 ) -> dict[str, Any]:
     """Prompt-generating function to pass to `IterableDataset.map`."""
     prompts = []
-    templates = list(prompter.templates.values())
+    templates = sorted(list(prompter.templates.values()), key=lambda t: t.name)
 
     def qa_cat(q: str, a: str) -> str:
         # if the jinja template already adds whitespace, don't add more
@@ -182,6 +190,7 @@ def qa_cat(q: str, a: str) -> str:
     # If they're not, we need to convert them with index(). label_choices is guaranteed
     # to be sorted (see above).
     return dict(
+        row_id=example["row_id"],
         label=label_choices.index(label),
         prompts=prompts,
         template_names=[template.name for template in templates],
diff --git a/ccs/metrics/__init__.py b/ccs/metrics/__init__.py
index 7fb21450..b07f67ff 100644
--- a/ccs/metrics/__init__.py
+++ b/ccs/metrics/__init__.py
@@ -1,6 +1,6 @@
 from .accuracy import accuracy_ci
 from .calibration import CalibrationError, CalibrationEstimate
-from .eval import EvalResult, evaluate_preds, to_one_hot
+from .eval import EvalResult, evaluate_preds, get_logprobs, to_one_hot
 from .roc_auc import RocAucResult, roc_auc, roc_auc_ci
 
 __all__ = [
@@ -9,6 +9,7 @@
     "CalibrationEstimate",
     "EvalResult",
     "evaluate_preds",
+    "get_logprobs",
     "roc_auc",
     "roc_auc_ci",
     "to_one_hot",
diff --git a/ccs/metrics/eval.py b/ccs/metrics/eval.py
index 6bd31d1e..efdca332 100644
--- a/ccs/metrics/eval.py
+++ b/ccs/metrics/eval.py
@@ -2,6 +2,7 @@
 from typing import Any, Literal
 
 import torch
+import torch.nn.functional as F
 from einops import repeat
 from torch import Tensor
 
@@ -49,6 +50,29 @@ def to_dict(self, prefix: str = "") -> dict[str, Any]:
         }
 
 
+def get_logprobs(
+    y_logits: Tensor, ensembling: Literal["none", "partial", "full"] = "none"
+) -> Tensor:
+    """
+    Get the class probabilities from a tensor of logits.
+    Args:
+        y_logits: Predicted log-odds of the positive class, tensor of shape (n, v, c).
+    Returns:
+        Tensor of logprobs: If ensemble is "none" or "partial", tensor of shape (n, v).
+            If ensemble is "full", tensor of shape (n,).
+    """
+    assert y_logits.shape[-1] == 2, "Logits must be binary."
+    if ensembling == "full":
+        y_logits = y_logits.mean(dim=1)
+
+    y_logits = (
+        y_logits[..., 1]
+        if ensembling == "none"
+        else y_logits[..., 1] - y_logits[..., 0]
+    )
+    return F.logsigmoid(y_logits)
+
+
 def evaluate_preds(
     y_true: Tensor,
     y_logits: Tensor,
diff --git a/ccs/promptsource/templates.py b/ccs/promptsource/templates.py
index 4d549abf..37459277 100644
--- a/ccs/promptsource/templates.py
+++ b/ccs/promptsource/templates.py
@@ -134,7 +134,7 @@ def get_fixed_answer_choices_list(self):
         else:
             return None
 
-    def apply(self, example, truncate=True, highlight_variables=False):
+    def apply(self, example, truncate=False, highlight_variables=False):
         """
         Creates a prompt by applying this template to an example
 
diff --git a/ccs/run.py b/ccs/run.py
index efa8a392..6da1e299 100644
--- a/ccs/run.py
+++ b/ccs/run.py
@@ -31,6 +31,16 @@
 )
 
 
+@dataclass
+class LayerData:
+    hiddens: Tensor
+    labels: Tensor
+    lm_preds: Tensor | None
+    texts: list[list[str]]  # (n, v)
+    row_ids: Tensor  # (n,)
+    variant_ids: list[list[str]]  # (n, v)
+
+
 @dataclass
 class Run(ABC, Serializable):
     data: Extract
@@ -46,6 +56,16 @@ class Run(ABC, Serializable):
     prompt_indices: tuple[int, ...] = ()
     """The indices of the prompt templates to use. If empty, all prompts are used."""
 
+    save_logprobs: bool = field(default=False, to_dict=False)
+    """ saves logprobs.pt containing
+        {<dsname>: {"row_ids": [n,], "variant_ids": [n, v],
+            "labels": [n,], "texts": [n, v],
+            "lm": {"none": [n, v], "full": [n,]},
+            "reporter": {<layer>: {"none": [n, v], "full": [n,]}},
+            "lr": {<layer>: {<inlp_iter>: {"none": [n, v], "full": [n,]}}}
+        }}
+    """
+
     concatenated_layer_offset: int = 0
     debug: bool = False
     min_gpu_mem: int | None = None  # in bytes
@@ -98,7 +118,7 @@ def execute(
 
         devices = select_usable_devices(self.num_gpus, min_memory=self.min_gpu_mem)
         num_devices = len(devices)
-        func: Callable[[int], dict[str, pd.DataFrame]] = partial(
+        func: Callable[[int], tuple[dict[str, pd.DataFrame], dict]] = partial(
             self.apply_to_layer, devices=devices, world_size=num_devices
         )
         self.apply_to_layers(func=func, num_devices=num_devices)
@@ -106,7 +126,7 @@ def execute(
     @abstractmethod
     def apply_to_layer(
         self, layer: int, devices: list[str], world_size: int
-    ) -> dict[str, pd.DataFrame]:
+    ) -> tuple[dict[str, pd.DataFrame], dict]:
         """Train or eval a reporter on a single layer."""
 
     def make_reproducible(self, seed: int):
@@ -125,24 +145,41 @@ def get_device(self, devices, world_size: int) -> str:
 
     def prepare_data(
         self, device: str, layer: int, split_type: Literal["train", "val"]
-    ) -> dict[str, tuple[Tensor, Tensor, Tensor | None]]:
+    ) -> dict[str, LayerData]:
         """Prepare data for the specified layer and split type."""
         out = {}
 
         for ds_name, ds in self.datasets:
             key = select_split(ds, split_type)
 
-            split = ds[key].with_format("torch", device=device, dtype=torch.int16)
-            labels = assert_type(Tensor, split["label"])
+            hidden_cols = [
+                col for col in ds[key].column_names if col.startswith("hidden_")
+            ]
+            split = ds[key].with_format(
+                "torch", device=device, dtype=torch.int16, columns=hidden_cols
+            )
+            # hiddens shape: (num_examples, num_variants, hidden_d)
             hiddens = int16_to_float32(assert_type(Tensor, split[f"hidden_{layer}"]))
             if self.prompt_indices:
                 hiddens = hiddens[:, self.prompt_indices]
 
-            with split.formatted_as("torch", device=device):
-                has_preds = "model_logits" in split.features
-                lm_preds = split["model_logits"] if has_preds else None
+            # convert the remaining columns to torch
+            split = split.with_format("torch", device=device)
+            labels = assert_type(Tensor, split["label"])
 
-            out[ds_name] = (hiddens, labels.to(hiddens.device), lm_preds)
+            if "lm_log_odds" in split.column_names:
+                lm_preds = assert_type(Tensor, split["lm_log_odds"])
+            else:
+                lm_preds = None
+
+            out[ds_name] = LayerData(
+                hiddens=hiddens,
+                labels=labels,
+                lm_preds=lm_preds,
+                texts=split["texts"],
+                row_ids=assert_type(Tensor, split["row_id"]),
+                variant_ids=split["variant_ids"],
+            )
 
         return out
 
@@ -155,7 +192,7 @@ def concatenate(self, layers):
 
     def apply_to_layers(
         self,
-        func: Callable[[int], dict[str, pd.DataFrame]],
+        func: Callable[[int], tuple[dict[str, pd.DataFrame], dict]],
         num_devices: int,
     ):
         """Apply a function to each layer of the datasets in parallel
@@ -178,11 +215,19 @@ def apply_to_layers(
         with ctx.Pool(num_devices) as pool:
             mapper = pool.imap_unordered if num_devices > 1 else map
             df_buffers = defaultdict(list)
+            logprobs_dicts = defaultdict(dict)
 
             try:
-                for df_dict in tqdm(mapper(func, layers), total=len(layers)):
+                for df_dict, logprobs_dict in tqdm(
+                    mapper(func, layers), total=len(layers)
+                ):
+                    # get arbitrary value
+                    df_ = next(iter(df_dict.values()))
+                    layer = df_["layer"].iloc[0]
                     for k, v in df_dict.items():
                         df_buffers[k].append(v)
+                    for k, v in logprobs_dict.items():
+                        logprobs_dicts[k][layer] = logprobs_dict[k]
             finally:
                 # Make sure the CSVs are written even if we crash or get interrupted
                 for name, dfs in df_buffers.items():
@@ -190,3 +235,24 @@ def apply_to_layers(
                     df.round(4).to_csv(self.out_dir / f"{name}.csv", index=False)
                 if self.debug:
                     save_debug_log(self.datasets, self.out_dir)
+                if self.save_logprobs:
+                    save_dict = defaultdict(dict)
+                    for ds_name, logprobs_dict in logprobs_dicts.items():
+                        save_dict[ds_name]["row_ids"] = logprobs_dict[layers[0]][
+                            "row_ids"
+                        ]
+                        save_dict[ds_name]["texts"] = logprobs_dict[layers[0]]["texts"]
+                        save_dict[ds_name]["labels"] = logprobs_dict[layers[0]][
+                            "labels"
+                        ]
+                        save_dict[ds_name]["lm"] = logprobs_dict[layers[0]]["lm"]
+                        save_dict[ds_name]["reporter"] = dict()
+                        save_dict[ds_name]["lr"] = dict()
+                        for layer, logprobs_dict_by_mode in logprobs_dict.items():
+                            save_dict[ds_name]["reporter"][
+                                layer
+                            ] = logprobs_dict_by_mode["reporter"]
+                            save_dict[ds_name]["lr"][layer] = logprobs_dict_by_mode[
+                                "lr"
+                            ]
+                    torch.save(dict(save_dict), self.out_dir / "logprobs.pt")
diff --git a/ccs/training/classifier.py b/ccs/training/classifier.py
index 148da939..c9373c54 100644
--- a/ccs/training/classifier.py
+++ b/ccs/training/classifier.py
@@ -63,7 +63,7 @@ def fit(
         x: Tensor,
         y: Tensor,
         *,
-        l2_penalty: float = 0.0,
+        l2_penalty: float = 0.001,
         max_iter: int = 10_000,
     ) -> float:
         """Fits the model to the input data using L-BFGS with L2 regularization.
diff --git a/ccs/training/supervised.py b/ccs/training/supervised.py
index d2eef5f7..e629a84e 100644
--- a/ccs/training/supervised.py
+++ b/ccs/training/supervised.py
@@ -2,19 +2,20 @@
 from einops import rearrange, repeat
 
 from ..metrics import to_one_hot
+from ..run import LayerData
 from .classifier import Classifier
 
 
 def train_supervised(
-    data: dict[str, tuple], device: str, mode: str
+    data: dict[str, LayerData], device: str, mode: str
 ) -> list[Classifier]:
     Xs, train_labels = [], []
 
-    for train_h, labels, _ in data.values():
-        (_, v, k, _) = train_h.shape
-        train_h = rearrange(train_h, "n v k d -> (n v k) d")
+    for train_data in data.values():
+        (_, v, k, _) = train_data.hiddens.shape
+        train_h = rearrange(train_data.hiddens, "n v k d -> (n v k) d")
 
-        labels = repeat(labels, "n -> (n v)", v=v)
+        labels = repeat(train_data.labels, "n -> (n v)", v=v)
         labels = to_one_hot(labels, k).flatten()
 
         Xs.append(train_h)
diff --git a/ccs/training/train.py b/ccs/training/train.py
index 3d00b54c..7a7889f6 100644
--- a/ccs/training/train.py
+++ b/ccs/training/train.py
@@ -12,7 +12,7 @@
 from simple_parsing.helpers.serialization import save
 
 from ..extraction import Extract
-from ..metrics import evaluate_preds, to_one_hot
+from ..metrics import evaluate_preds, get_logprobs, to_one_hot
 from ..run import Run
 from ..training.supervised import train_supervised
 from ..utils.typing import assert_type
@@ -63,7 +63,7 @@ def apply_to_layer(
         layer: int,
         devices: list[str],
         world_size: int,
-    ) -> dict[str, pd.DataFrame]:
+    ) -> tuple[dict[str, pd.DataFrame], dict]:
         """Train a single reporter on a single layer."""
 
         self.make_reproducible(seed=self.net.seed + layer)
@@ -72,16 +72,16 @@ def apply_to_layer(
         train_dict = self.prepare_data(device, layer, "train")
         val_dict = self.prepare_data(device, layer, "val")
 
-        (first_train_h, train_gt, _), *rest = train_dict.values()
-        (_, v, k, d) = first_train_h.shape
-        if not all(other_h.shape[-1] == d for other_h, _, _ in rest):
+        first_train_data, *rest = train_dict.values()
+        (_, v, k, d) = first_train_data.hiddens.shape
+        if not all(other.hiddens.shape[-1] == d for other in rest):
             raise ValueError("All datasets must have the same hidden state size")
 
         # For a while we did support datasets with different numbers of classes, but
         # we reverted this once we switched to ConceptEraser. There are a few options
         # for re-enabling it in the future but they are somewhat complex and it's not
         # clear that it's worth it.
-        if not all(other_h.shape[-2] == k for other_h, _, _ in rest):
+        if not all(other.hiddens.shape[-2] == k for other in rest):
             raise ValueError("All datasets must have the same number of classes")
 
         reporter_dir, lr_dir = self.create_models_dir(assert_type(Path, self.out_dir))
@@ -91,9 +91,9 @@ def apply_to_layer(
             assert len(train_dict) == 1, "CCS only supports single-task training"
 
             reporter = CcsReporter(self.net, d, device=device, num_variants=v)
-            train_loss = reporter.fit(first_train_h)
-            labels = repeat(to_one_hot(train_gt, k), "n k -> n v k", v=v)
-            reporter.platt_scale(labels, first_train_h)
+            train_loss = reporter.fit(first_train_data.hiddens)
+            labels = repeat(to_one_hot(first_train_data.labels, k), "n k -> n v k", v=v)
+            reporter.platt_scale(labels, first_train_data.hiddens)
 
         elif isinstance(self.net, EigenFitterConfig):
             fitter = EigenFitter(
@@ -101,16 +101,20 @@ def apply_to_layer(
             )
 
             hidden_list, label_list = [], []
-            for ds_name, (train_h, train_gt, _) in train_dict.items():
-                (_, v, _, _) = train_h.shape
+            for ds_name, train_data in train_dict.items():
+                (_, v, _, _) = train_data.hiddens.shape
 
                 # Datasets can have different numbers of variants, so we need to
                 # flatten them here before concatenating
-                hidden_list.append(rearrange(train_h, "n v k d -> (n v k) d"))
+                hidden_list.append(
+                    rearrange(train_data.hiddens, "n v k d -> (n v k) d")
+                )
                 label_list.append(
-                    to_one_hot(repeat(train_gt, "n -> (n v)", v=v), k).flatten()
+                    to_one_hot(
+                        repeat(train_data.labels, "n -> (n v)", v=v), k
+                    ).flatten()
                 )
-                fitter.update(train_h)
+                fitter.update(train_data.hiddens)
 
             reporter = fitter.fit_streaming()
             reporter.platt_scale(
@@ -135,59 +139,108 @@ def apply_to_layer(
         else:
             lr_models = []
 
-        row_bufs = defaultdict(list)
-        for ds_name in val_dict:
-            val_h, val_gt, val_lm_preds = val_dict[ds_name]
-            train_h, train_gt, train_lm_preds = train_dict[ds_name]
-            meta = {"dataset": ds_name, "layer": layer}
-
-            val_credences = reporter(val_h)
-            train_credences = reporter(train_h)
-            for mode in ("none", "partial", "full"):
-                row_bufs["eval"].append(
-                    {
-                        **meta,
-                        "ensembling": mode,
-                        **evaluate_preds(val_gt, val_credences, mode).to_dict(),
-                        "train_loss": train_loss,
-                    }
-                )
-
-                row_bufs["train_eval"].append(
-                    {
-                        **meta,
-                        "ensembling": mode,
-                        **evaluate_preds(train_gt, train_credences, mode).to_dict(),
-                        "train_loss": train_loss,
-                    }
-                )
-
-                if val_lm_preds is not None:
-                    row_bufs["lm_eval"].append(
-                        {
-                            **meta,
-                            "ensembling": mode,
-                            **evaluate_preds(val_gt, val_lm_preds, mode).to_dict(),
-                        }
+        with torch.no_grad():
+            out_logprobs = defaultdict(dict)
+            row_bufs = defaultdict(list)
+            for ds_name in val_dict:
+                val, train = val_dict[ds_name], train_dict[ds_name]
+                meta = {"dataset": ds_name, "layer": layer}
+
+                if self.save_logprobs:
+                    out_logprobs[ds_name] = dict(
+                        row_ids=val.row_ids.cpu(),
+                        variant_ids=val.variant_ids,
+                        texts=val.texts,
+                        labels=val.labels.cpu(),
+                        lm=dict(),
+                        lr=dict(),
+                        reporter=dict(),
                     )
 
-                if train_lm_preds is not None:
-                    row_bufs["train_lm_eval"].append(
+                val_credences = reporter(val.hiddens)
+                train_credences = reporter(train.hiddens)
+                for mode in ("none", "partial", "full"):
+                    row_bufs["eval"].append(
                         {
                             **meta,
                             "ensembling": mode,
-                            **evaluate_preds(train_gt, train_lm_preds, mode).to_dict(),
+                            **evaluate_preds(val.labels, val_credences, mode).to_dict(),
+                            "train_loss": train_loss,
                         }
                     )
+                    if self.save_logprobs:
+                        out_logprobs[ds_name]["reporter"][mode] = get_logprobs(
+                            val_credences, mode
+                        ).cpu()
 
-                for i, model in enumerate(lr_models):
-                    row_bufs["lr_eval"].append(
+                    row_bufs["train_eval"].append(
                         {
                             **meta,
                             "ensembling": mode,
-                            "inlp_iter": i,
-                            **evaluate_preds(val_gt, model(val_h), mode).to_dict(),
+                            **evaluate_preds(
+                                train.labels, train_credences, mode
+                            ).to_dict(),
+                            "train_loss": train_loss,
                         }
                     )
 
-        return {k: pd.DataFrame(v) for k, v in row_bufs.items()}
+                    if val.lm_preds is not None:
+                        row_bufs["lm_eval"].append(
+                            {
+                                **meta,
+                                "ensembling": mode,
+                                **evaluate_preds(
+                                    val.labels, val.lm_preds, mode
+                                ).to_dict(),
+                            }
+                        )
+                        if self.save_logprobs:
+                            out_logprobs[ds_name]["lm"][mode] = get_logprobs(
+                                val.lm_preds, mode
+                            ).cpu()
+
+                    if train.lm_preds is not None:
+                        row_bufs["train_lm_eval"].append(
+                            {
+                                **meta,
+                                "ensembling": mode,
+                                **evaluate_preds(
+                                    train.labels, train.lm_preds, mode
+                                ).to_dict(),
+                            }
+                        )
+
+                    if self.save_logprobs:
+                        out_logprobs[ds_name]["lr"][mode] = dict()
+
+                    for i, model in enumerate(lr_models):
+                        model.eval()
+                        val_credences = model(val.hiddens)
+                        train_credences = model(train.hiddens)
+
+                        if self.save_logprobs:
+                            out_logprobs[ds_name]["lr"][mode][i] = get_logprobs(
+                                val_credences, mode
+                            ).cpu()
+                        row_bufs["lr_eval"].append(
+                            {
+                                **meta,
+                                "ensembling": mode,
+                                "inlp_iter": i,
+                                **evaluate_preds(
+                                    val.labels, val_credences, mode
+                                ).to_dict(),
+                            }
+                        )
+                        row_bufs["train_lr_eval"].append(
+                            {
+                                **meta,
+                                "ensembling": mode,
+                                "inlp_iter": i,
+                                **evaluate_preds(
+                                    train.labels, train_credences, mode
+                                ).to_dict(),
+                            }
+                        )
+
+        return {k: pd.DataFrame(v) for k, v in row_bufs.items()}, out_logprobs

From 26cf3b37a89049f86fee1d2d39c0c98b5f2ccd3d Mon Sep 17 00:00:00 2001
From: Alex Mallen <atmallen8@gmail.com>
Date: Fri, 10 Nov 2023 23:40:28 +0000
Subject: [PATCH 64/77] remove empty folder:

---
 comparison-sweeps | 1 -
 1 file changed, 1 deletion(-)
 delete mode 160000 comparison-sweeps

diff --git a/comparison-sweeps b/comparison-sweeps
deleted file mode 160000
index f4ed884b..00000000
--- a/comparison-sweeps
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit f4ed884b59c99012c80b972d2a02c660b39c90cb

From c17a37e097ae91d7e6a52663230349971464def9 Mon Sep 17 00:00:00 2001
From: Alex Mallen <atmallen8@gmail.com>
Date: Mon, 13 Nov 2023 17:46:41 +0000
Subject: [PATCH 65/77] add embedding layer; fix lm_ log_odds computation; add
 prefix space

---
 ccs/__init__.py              |  4 ++++
 ccs/extraction/extraction.py | 34 +++++++++++++++++++---------------
 2 files changed, 23 insertions(+), 15 deletions(-)

diff --git a/ccs/__init__.py b/ccs/__init__.py
index ce69da0d..f9007c4e 100644
--- a/ccs/__init__.py
+++ b/ccs/__init__.py
@@ -1,5 +1,7 @@
 from .extraction import Extract, extract_hiddens
 from .training import EigenFitter, EigenFitterConfig
+from .training.train import Elicit
+from .evaluation import Eval
 from .truncated_eigh import truncated_eigh
 
 __all__ = [
@@ -7,5 +9,7 @@
     "EigenFitterConfig",
     "extract_hiddens",
     "Extract",
+    "Elicit",
+    "Eval",
     "truncated_eigh",
 ]
diff --git a/ccs/extraction/extraction.py b/ccs/extraction/extraction.py
index f6e0b4b9..741d1026 100644
--- a/ccs/extraction/extraction.py
+++ b/ccs/extraction/extraction.py
@@ -137,8 +137,9 @@ def __post_init__(self, layer_stride: int):
             config = assert_type(
                 PretrainedConfig, AutoConfig.from_pretrained(self.model)
             )
-            layer_range = range(1, config.num_hidden_layers, layer_stride)
-            self.layers = tuple(layer_range)
+            # Note that we always include 0 which is the embedding layer
+            layer_range = range(1, config.num_hidden_layers + 1, layer_stride)
+            self.layers = (0,) + tuple(layer_range)
 
     def explode(self) -> list["Extract"]:
         """Explode this config into a list of configs, one for each layer."""
@@ -198,7 +199,7 @@ def extract_hiddens(
         seed=cfg.seed,
     )
 
-    layer_indices = cfg.layers or tuple(range(1, model.config.num_hidden_layers))
+    layer_indices = cfg.layers or tuple(range(model.config.num_hidden_layers + 1))
 
     global_max_examples = cfg.max_examples[0 if split_type == "train" else 1]
 
@@ -263,14 +264,17 @@ def extract_hiddens(
                 if is_enc_dec:
                     answer = labels = assert_type(Tensor, encoding.labels)
                 else:
-                    encoding2 = tokenizer(
-                        choice["answer"],
-                        # Don't include [CLS] and [SEP] in the answer
-                        add_special_tokens=False,
-                        return_tensors="pt",
-                    ).to(device)
-
-                    answer = assert_type(Tensor, encoding2.input_ids)
+                    a_id = tokenizer.encode(" " + choice["answer"], add_special_tokens=False)
+
+                    # the Llama tokenizer splits off leading spaces
+                    if tokenizer.decode(a_id[0]).strip() == "":
+                        a_id_without_space = tokenizer.encode(
+                            choice, add_special_tokens=False
+                        )
+                        assert a_id_without_space == a_id[1:]
+                        a_id = a_id_without_space
+
+                    answer = torch.tensor([a_id], device=device)
                     labels = (
                         # -100 is the mask token
                         torch.cat([torch.full_like(ids, -100), answer], dim=-1)
@@ -293,13 +297,13 @@ def extract_hiddens(
 
                 # Compute the log probability of the answer tokens if available
                 if has_lm_preds:
-                    logprob = -assert_type(Tensor, outputs.loss)
+                    logprob = -assert_type(Tensor, outputs.loss).to(torch.float32)
                     # Convert logprob to logodds to be consistent with reporters
                     # Because we went through logprobs, logodds corresponding to
                     # probs near 1 will be somewhat imprecise
                     # log(p/(1-p)) = log(p) - log(1-p) = logp - log(1 - exp(logp))
                     lm_log_odds[i, j] = logprob - torch.log1p(-logprob.exp())
-
+                    
                 hiddens = (
                     outputs.get("decoder_hidden_states") or outputs["hidden_states"]
                 )
@@ -339,7 +343,7 @@ def extract_hiddens(
             **hidden_dict,
         )
         if has_lm_preds:
-            out_record["lm_log_odds"] = lm_log_odds.log_softmax(dim=-1)
+            out_record["lm_log_odds"] = lm_log_odds
 
         assert out_record["variant_ids"] == sorted(out_record["variant_ids"])
         num_yielded += 1
@@ -377,7 +381,7 @@ def hidden_features(cfg: Extract) -> tuple[DatasetInfo, Features]:
     if num_dropped:
         print(f"Dropping {num_dropped} non-multiple choice templates")
 
-    layer_indices = cfg.layers or tuple(range(1, model_cfg.num_hidden_layers))
+    layer_indices = cfg.layers or tuple(range(model_cfg.num_hidden_layers + 1))
     layer_cols = {
         f"hidden_{layer}": Array3D(
             dtype="int16",

From e7514050d034f037852ac3aa618c9ab8ebc4055f Mon Sep 17 00:00:00 2001
From: Alex Mallen <atmallen8@gmail.com>
Date: Tue, 14 Nov 2023 17:45:50 +0000
Subject: [PATCH 66/77] make ccs default

---
 ccs/training/train.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ccs/training/train.py b/ccs/training/train.py
index 7a7889f6..a0510637 100644
--- a/ccs/training/train.py
+++ b/ccs/training/train.py
@@ -26,7 +26,7 @@ class Elicit(Run):
     """Full specification of a reporter training run."""
 
     net: FitterConfig = subgroups(
-        {"ccs": CcsConfig, "eigen": EigenFitterConfig}, default="eigen"  # type: ignore
+        {"ccs": CcsConfig, "eigen": EigenFitterConfig}, default="ccs"  # type: ignore
     )
     """Config for building the reporter network."""
 

From 6e07819099c44f0ca00c197186e32d27fc1115c9 Mon Sep 17 00:00:00 2001
From: Alex Mallen <atmallen8@gmail.com>
Date: Tue, 14 Nov 2023 19:53:03 +0000
Subject: [PATCH 67/77] default meanonly norm

---
 ccs/training/ccs_reporter.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/ccs/training/ccs_reporter.py b/ccs/training/ccs_reporter.py
index ce20fee3..11ed842c 100644
--- a/ccs/training/ccs_reporter.py
+++ b/ccs/training/ccs_reporter.py
@@ -43,7 +43,7 @@ class CcsConfig(FitterConfig):
     function 1.0*consistency_squared + 0.5*prompt_var.
     """
     loss_dict: dict[str, float] = field(default_factory=dict, init=False)
-    norm: Literal["leace", "burns"] = "leace"
+    norm: Literal["leace", "burns", "meanonly"] = "meanonly"
     num_layers: int = 1
     """The number of layers in the MLP."""
     pre_ln: bool = False
@@ -209,6 +209,8 @@ def fit(self, hiddens: Tensor) -> float:
 
         if self.config.norm == "burns":
             self.norm = BurnsNorm()
+        elif self.config.norm == "meanonly":
+            self.norm = BurnsNorm(scale=False)
         else:
             fitter = LeaceFitter(d, 2 * v, dtype=x_neg.dtype, device=x_neg.device)
             fitter.update(

From 5815371b70496d49145f27a76d927273db3050ce Mon Sep 17 00:00:00 2001
From: Walter Laurito <lauritowal@yahoo.com>
Date: Sat, 31 Aug 2024 16:18:11 +0200
Subject: [PATCH 68/77] add additional authors, sort by number of commits

---
 joss/paper.md | 28 ++++++++++++++++++----------
 1 file changed, 18 insertions(+), 10 deletions(-)

diff --git a/joss/paper.md b/joss/paper.md
index 89ffd836..aa87652c 100644
--- a/joss/paper.md
+++ b/joss/paper.md
@@ -7,23 +7,24 @@ tags:
   - ai alignment
   - honest AI
 authors:
-  - name: Nora Belrose
-    affiliation: 1 
+  # sorted by num of commits
   - name: Walter Laurito
     corresponding: true
     affiliation: "2, 3"
-  - name: Alex Mallen
-    affiliation: "1, 7"
-  - name: Fabien Roger
-    affiliation: 4
+  - name: Nora Belrose
+  affiliation: 1 
   - name: Kay Kozaronek
     affiliation: 2
+  - name: Fabien Roger
+  affiliation: 4
+  - name: Alex Mallen
+    affiliation: "1, 7"
   - name: Christy Koh
     affiliation: 5
-  - name: Jonathan NG
-    affiliation: 2
   - name: James Chua
     affiliation: 1
+  - name: Jonathan NG
+    affiliation: 2
   - name: Alexander Wan
     affiliation: 5
   - name: Reagan Lee
@@ -34,10 +35,15 @@ authors:
     affiliation: "1, 6"
   - name: Augustas Macijauskas
     affiliation: 8
-  - name: Waree Sethapun
-    affiliation: 9
   - name: Eric Mungai Kinuthia
     affiliation: 1
+  - name: Marius PL
+    affiliation: 10
+  - name: Waree Sethapun
+    affiliation: 9
+  - name: Kaarel Hänni
+    affiliation: 10
+
 affiliations:
  - name: EleutherAI
    index: 1
@@ -57,6 +63,8 @@ affiliations:
    index: 8
  - name: Princeton University
    index: 9
+ - name: Independent
+   index: 10
 date: 11 08 2023
 bibliography: paper.bib
 

From a2d53952e064786e62f6726700e5caab48385780 Mon Sep 17 00:00:00 2001
From: Walter Laurito <lauritowal@yahoo.com>
Date: Sat, 31 Aug 2024 16:36:45 +0200
Subject: [PATCH 69/77] add cleare contribution guidelines

---
 README.md | 64 +++++++++++++++++++++++++++++++++++++++----------------
 1 file changed, 46 insertions(+), 18 deletions(-)

diff --git a/README.md b/README.md
index 10b8166a..b0287d9a 100644
--- a/README.md
+++ b/README.md
@@ -21,7 +21,19 @@ Our code is based on [PyTorch](http://pytorch.org)
 and [Huggingface Transformers](https://huggingface.co/docs/transformers/index). We test the code on Python 3.10 and
 3.11.
 
-First install the package with `pip install -e .` in the root directory. Use `pip install -e .[dev]` if you'd like to contribute to the project (see **Development** section below). This should install all the necessary dependencies.
+First, create a virtual environment by using e.g. conda:
+
+```
+conda create -n ccs python==3.10
+conda activate ccs
+```
+
+Clone the repository:
+```
+git clone https://github.com/EleutherAI/ccs.git
+```
+
+Next, install the package with `python -m pip install -e .` in the root directory. Use `python -m pip install -e .[dev]` if you'd like to contribute to the project (see **Development** section below). This should install all the necessary dependencies.
 
 To fit reporters for the HuggingFace model `model` and dataset `dataset`, just run:
 
@@ -70,27 +82,41 @@ The hidden states resulting from `ccs elicit` are cached as a HuggingFace datase
 every time we want to train a probe. The cache is stored in the same place as all other HuggingFace datasets, which is
 usually `~/.cache/huggingface/datasets`.
 
-## Development
+## Contribution Guidelines
 
-Use `pip install pre-commit && pre-commit install` in the root folder before your first commit.
+If you work on a new feature / fix or some other code task, make sure to create an issue and assign it to yourself.
+Maybe, even share it in the elk channel of Eleuther's Discord with a small note. In this way, others know you are
+working on the issue and people won't do the same thing twice 👍 Also others can contact you easily.
 
-### Devcontainer
+### Submitting a Pull-Requests
+We welcome PRs to our libraries. They're an efficient way to include your fixes or improvements in our next release. Please follow these guidelines:
 
-[
-![Open in Remote - Containers](
-https://img.shields.io/static/v1?label=Remote%20-%20Containers&message=Open&color=blue&logo=visualstudiocode
-)
-](
-https://vscode.dev/redirect?url=vscode://ms-vscode-remote.remote-containers/cloneInVolume?url=https://github.com/EleutherAI/ccs
-)
+- Focus on either functionality changes OR widespread style issues, not both.
+- Add tests for new or modified functionality if it makes sense.
+- Address a single issue or feature with minimal code changes.
+- Include relevant documentation in the repo or on our docs site.
 
-### Run tests
+#### "fork-and-pull" Git workflow:
+
+- Fork the repository to your Github account.
+- Clone the project to your local machine.
+- Create a new branch with a concise, descriptive name.
+- Make and commit your changes to our neww branch.
+- Follow any repo-specific formatting and testing guidelines (see next section)
+- Push the changes to your fork.
+- Open a PR in our repository, using the PR template for efficient review.
+
+
+#### For each commit
+1. Use `python -m pip install pre-commit && pre-commit install` in the root folder before your first commit.
+
+2. Run tests
 
 ```bash
 pytest
 ```
 
-### Run type checking
+3. Run type checking
 
 We use [pyright](https://github.com/microsoft/pyright), which is built into the VSCode editor. If you'd like to run it
 as a standalone tool, it requires a [nodejs installation.](https://nodejs.org/en/download/)
@@ -99,7 +125,7 @@ as a standalone tool, it requires a [nodejs installation.](https://nodejs.org/en
 pyright
 ```
 
-### Run the linter
+4. Run the linter
 
 We use [ruff](https://beta.ruff.rs/docs/). It is installed as a pre-commit hook, so you don't have to run it manually.
 If you want to run it manually, you can do so with:
@@ -108,8 +134,10 @@ If you want to run it manually, you can do so with:
 ruff . --fix
 ```
 
-### Contributing to this repository
+### Issues
 
-If you work on a new feature / fix or some other code task, make sure to create an issue and assign it to yourself (
-Maybe, even share it in the elk channel of Eleuther's Discord with a small note). In this way, others know you are
-working on the issue and people won't do the same thing twice 👍 Also others can contact you easily.
+Issues serve three main purposes: reporting library problems, requesting new features, and discussing potential changes before creating a Pull Request (PR). If you encounter a problem, first check if an existing Issue addresses it. If so, add your own reproduction information to that Issue instead of creating a new one. This approach prevents duplicate reports and helps maintainers understand the problem's scope. Additionally, adding a reaction (like a thumbs-up) to an existing Issue signals to maintainers that the problem affects multiple users, which can influence prioritization.
+
+### Discussion and Contact
+
+If you have additional questions you ask them in the elk channel of Eleuther's Discord https://discord.gg/zBGx3azzUn 

From 7e6f9b5c8cee23c98e05a5a2df1b0bb12519ee23 Mon Sep 17 00:00:00 2001
From: Walter Laurito <lauritowal@yahoo.com>
Date: Sat, 31 Aug 2024 16:37:15 +0200
Subject: [PATCH 70/77] rename title

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index b0287d9a..4b22e58d 100644
--- a/README.md
+++ b/README.md
@@ -107,7 +107,7 @@ We welcome PRs to our libraries. They're an efficient way to include your fixes
 - Open a PR in our repository, using the PR template for efficient review.
 
 
-#### For each commit
+#### Before commiting
 1. Use `python -m pip install pre-commit && pre-commit install` in the root folder before your first commit.
 
 2. Run tests

From c48d69dab67a56f3ecf73c60d86ee076d5ef2383 Mon Sep 17 00:00:00 2001
From: Walter Laurito <lauritowal@yahoo.com>
Date: Sat, 31 Aug 2024 16:39:09 +0200
Subject: [PATCH 71/77] address minor changes

---
 LICENSE.md | 2 +-
 README.md  | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/LICENSE.md b/LICENSE.md
index ae64ec4f..3525d595 100644
--- a/LICENSE.md
+++ b/LICENSE.md
@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2023 EleutherAI
+Copyright (c) 2024 EleutherAI
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
diff --git a/README.md b/README.md
index 4b22e58d..587432c8 100644
--- a/README.md
+++ b/README.md
@@ -5,8 +5,8 @@
 Because language models are trained to predict the next token in naturally occurring text, they often reproduce common
 human errors and misconceptions, even when they "know better" in some sense. More worryingly, when models are trained to
 generate text that's rated highly by humans, they may learn to output false statements that human evaluators can't
-detect. We aim to circumvent this issue by directly [**eliciting latent knowledge
-**](https://docs.google.com/document/d/1WwsnJQstPq91_Yh-Ch2XRL8H_EpsnjrC1dwZXR37PC8/edit) (ELK) inside the activations
+detect. We aim to circumvent this issue by directly [eliciting latent knowledge
+](https://docs.google.com/document/d/1WwsnJQstPq91_Yh-Ch2XRL8H_EpsnjrC1dwZXR37PC8/edit) (ELK) inside the activations
 of a language model.
 
 Specifically, we're building on the **Contrastive Representation Clustering** (CRC) method described in the

From 850a108ceb363335183db6e8339723ae70adc49b Mon Sep 17 00:00:00 2001
From: Walter Laurito <lauritowal@yahoo.com>
Date: Sat, 31 Aug 2024 16:48:44 +0200
Subject: [PATCH 72/77] update Readme

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 587432c8..3578d517 100644
--- a/README.md
+++ b/README.md
@@ -33,7 +33,7 @@ Clone the repository:
 git clone https://github.com/EleutherAI/ccs.git
 ```
 
-Next, install the package with `python -m pip install -e .` in the root directory. Use `python -m pip install -e .[dev]` if you'd like to contribute to the project (see **Development** section below). This should install all the necessary dependencies.
+Next, install the package with `pip install -e .` in the root directory. Use `pip install -e .[dev]` if you'd like to contribute to the project (see **Development** section below). This should install all the necessary dependencies.
 
 To fit reporters for the HuggingFace model `model` and dataset `dataset`, just run:
 
@@ -108,7 +108,7 @@ We welcome PRs to our libraries. They're an efficient way to include your fixes
 
 
 #### Before commiting
-1. Use `python -m pip install pre-commit && pre-commit install` in the root folder before your first commit.
+1. Use `pip install pre-commit && pre-commit install` in the root folder before your first commit.
 
 2. Run tests
 

From 35b9fdce539cf972601e15bae8a41d4e657e70a9 Mon Sep 17 00:00:00 2001
From: Walter Laurito <lauritowal@yahoo.com>
Date: Sat, 31 Aug 2024 17:20:30 +0200
Subject: [PATCH 73/77] remove num gpus from tests

---
 tests/test_smoke_elicit.py | 2 --
 tests/test_smoke_eval.py   | 1 -
 2 files changed, 3 deletions(-)

diff --git a/tests/test_smoke_elicit.py b/tests/test_smoke_elicit.py
index 0b8b97a0..aaccba7d 100644
--- a/tests/test_smoke_elicit.py
+++ b/tests/test_smoke_elicit.py
@@ -16,7 +16,6 @@ def test_smoke_elicit_run_tiny_gpt2_ccs(tmp_path: Path):
             max_examples=(10, 10),
             # run on all layers, tiny-gpt only has 2 layers
         ),
-        num_gpus=2,
         min_gpu_mem=min_mem,
         net=CcsConfig(),
         out_dir=tmp_path,
@@ -47,7 +46,6 @@ def test_smoke_elicit_run_tiny_gpt2_eigen(tmp_path: Path):
             max_examples=(10, 10),
             # run on all layers, tiny-gpt only has 2 layers
         ),
-        num_gpus=2,
         min_gpu_mem=min_mem,
         net=EigenFitterConfig(),
         out_dir=tmp_path,
diff --git a/tests/test_smoke_eval.py b/tests/test_smoke_eval.py
index 19e1257c..c4bad941 100644
--- a/tests/test_smoke_eval.py
+++ b/tests/test_smoke_eval.py
@@ -32,7 +32,6 @@ def setup_elicit(
             max_examples=(10, 10),
             # run on all layers, tiny-gpt only has 2 layers
         ),
-        num_gpus=2,
         min_gpu_mem=min_mem,
         net=CcsConfig() if is_ccs else EigenFitterConfig(),
         out_dir=tmp_path,

From ce5316d1d97f708e53411fe0702ce3252705c98d Mon Sep 17 00:00:00 2001
From: Walter Laurito <lauritowal@yahoo.com>
Date: Sat, 31 Aug 2024 17:42:40 +0200
Subject: [PATCH 74/77] add more information about VINC

---
 ccs.lock | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 ccs.lock

diff --git a/ccs.lock b/ccs.lock
new file mode 100644
index 00000000..e69de29b

From fa5a77f6c5118e107bf553674c43e85a5f822891 Mon Sep 17 00:00:00 2001
From: Walter Laurito <lauritowal@yahoo.com>
Date: Sat, 31 Aug 2024 19:27:35 +0200
Subject: [PATCH 75/77] add longer introduction + improve some parts of the
 paper

---
 joss/paper.bib | 80 ++++++++++++++++++++++++++++++++++++++++++++++++++
 joss/paper.md  | 33 ++++++++++++++-------
 2 files changed, 102 insertions(+), 11 deletions(-)

diff --git a/joss/paper.bib b/joss/paper.bib
index 60ad6100..c1839c94 100644
--- a/joss/paper.bib
+++ b/joss/paper.bib
@@ -11,4 +11,84 @@ @misc{christiano2021
 	howpublished = {\url{https://docs.google.com/document/d/1WwsnJQstPq91_Yh-Ch2XRL8H_EpsnjrC1dwZXR37PC8/}},
 	year = {December 2021},
 	note = {[Accessed 11-08-2023]},
+}
+
+@article{weidinger2021ethical,
+  title={Ethical and social risks of harm from language models},
+  author={Weidinger, Laura and Mellor, John and Rauh, Moritz and Griffin, Conor and Uesato, Jonathan and Huang, Po-Sen and Cheng, Michael and Glaese, Matthew and Balle, Borja and Kasirzadeh, Atoosa and Kenton, Zachary},
+  year={2021},
+  journal={arXiv preprint arXiv:2112.04359}
+}
+
+@article{hendrycks2021unsolved,
+  title={Unsolved problems in ml safety},
+  author={Hendrycks, Dan and Carlini, Nicholas and Schulman, John and Steinhardt, Jacob},
+  year={2021},
+  journal={arXiv preprint arXiv:2109.13916}
+}
+
+@article{park2023ai,
+  title={AI deception: A survey of examples, risks, and potential solutions},
+  author={Park, Peter S and Goldstein, Samuel and O'Gara, Annette and Chen, Mingjie and Hendrycks, Dan},
+  year={2023},
+  journal={arXiv preprint arXiv:2308.14752}
+}
+
+@article{evans2021truthful,
+  title={Truthful AI: Developing and governing AI that does not lie},
+  author={Evans, Owain and Cotton-Barratt, Owen and Finnveden, Lars and Bales, Adam and Balwit, Amanda and Wills, Peter and Righetti, Luca and Saunders, William},
+  year={2021},
+  journal={arXiv preprint arXiv:2110.06674}
+}
+
+@article{li2022emergent,
+  title={Emergent world representations: Exploring a sequence model trained on a synthetic task},
+  author={Li, Kenneth and Hopkins, Aspen K and Bau, David and Vi{\'e}gas, Fernanda and Pfister, Hanspeter and Wattenberg, Martin},
+  journal={arXiv preprint arXiv:2210.13382},
+  year={2022}
+}
+
+@article{gurnee2023language,
+  title={Language models represent space and time},
+  author={Gurnee, Wes and Tegmark, Max},
+  journal={arXiv preprint arXiv:2310.02207},
+  year={2023}
+}
+
+
+@article{azaria2023internal,
+  title={The internal state of an llm knows when its lying},
+  author={Azaria, Amos and Mitchell, Tom},
+  journal={arXiv preprint arXiv:2304.13734},
+  year={2023}
+}
+
+@article{bubeck2023sparks,
+  title={Sparks of artificial general intelligence: Early experiments with gpt-4},
+  author={Bubeck, S{\'e}bastien and Chandrasekaran, Varun and Eldan, Ronen and Gehrke, Johannes and Horvitz, Eric and Kamar, Ece and Lee, Peter and Lee, Yin Tat and Li, Yuanzhi and Lundberg, Scott and others},
+  journal={arXiv preprint arXiv:2303.12712},
+  year={2023}
+}
+
+@article{alain2016understanding,
+  title={Understanding intermediate layers using linear classifier probes},
+  author={Alain, Guillaume and Bengio, Yoshua},
+  journal={arXiv preprint arXiv:1610.01644},
+  year={2016}
+}
+
+@misc{marks2023geometry,
+      title={The Geometry of Truth: Emergent Linear Structure in Large Language Model Representations of True/False Datasets}, 
+      author={Samuel Marks and Max Tegmark},
+      year={2023},
+      eprint={2310.06824},
+      archivePrefix={arXiv},
+      primaryClass={cs.AI}
+}
+
+@article{zou2023representation,
+  title={Representation engineering: A top-down approach to ai transparency},
+  author={Zou, Andy and Phan, Long and Chen, Sarah and Campbell, James and Guo, Phillip and Ren, Richard and Pan, Alexander and Yin, Xuwang and Mazeika, Mantas and Dombrowski, Ann-Kathrin and others},
+  journal={arXiv preprint arXiv:2310.01405},
+  year={2023}
 }
\ No newline at end of file
diff --git a/joss/paper.md b/joss/paper.md
index aa87652c..237cff10 100644
--- a/joss/paper.md
+++ b/joss/paper.md
@@ -2,12 +2,11 @@
 title: 'CCS-Lib: A Python package to elicit latent knowledge from LLMs'
 tags:
   - python
-  - machine leaarning
+  - machine learning
   - interpretability
   - ai alignment
   - honest AI
-authors:
-  # sorted by num of commits
+authors: # sorted by num of commits
   - name: Walter Laurito
     corresponding: true
     affiliation: "2, 3"
@@ -72,21 +71,33 @@ bibliography: paper.bib
 
 # Summary
 
-`ccs` is a library designed to elicit latent knowledge ([elk](`https://docs.google.com/document/d/1WwsnJQstPq91_Yh-Ch2XRL8H_EpsnjrC1dwZXR37PC8/`) [@christiano2021]) from language models. It includes implementations of both the original and an enhanced version of the CSS method, as well as an approach based on the CRC method [@burns2022]. Designed for researchers, `ccs` offers features such as multi-GPU support, integration with Huggingface, and continuous improvement by a dedicated group of people. The Eleuther AI Discord's `elk` channel provides a platform for collaboration and discussion related to the library and associated research.
+`ccs` is a library designed to elicit latent knowledge ([elk](`https://docs.google.com/document/d/1WwsnJQstPq91_Yh-Ch2XRL8H_EpsnjrC1dwZXR37PC8/`) [@christiano2021]) from language models. It includes implementations of both the original and an enhanced version of the CSS method, as well as an approach based on the Clustering—Top Principal Component (CRC-TPC) [@burns2022], called VINC. Designed for researchers, the `ccs` library offers features like multi-GPU support, integration with Huggingface and the training of supervised probes for comparisons. The Eleuther AI Discord's `elk` channel provides a platform for collaboration and discussion related to the library and associated research.
 
 # Statement of need
 
-Language models are proficient at predicting successive tokens in a sequence of text. However, they often inadvertently mirror human errors and misconceptions, even when equipped with the capability to "know better." This behavior becomes particularly concerning when models are trained to generate text that is highly rated by human evaluators, leading to the potential output of erroneous statements that may go undetected. Our solution is to directly elicit latent knowledge (([elk](`https://docs.google.com/document/d/1WwsnJQstPq91_Yh-Ch2XRL8H_EpsnjrC1dwZXR37PC8/`) [@christiano2021]) from within the activations of a language model to mitigate this challenge.
+The widespread adoption of language models in real-world applications presents significant challenges, particularly the potential generation of unreliable or inaccurate content [@weidinger2021ethical; @park2023ai; @evans2021truthful; @hendrycks2021unsolved]. A notable concern is that models fine-tuned on human preferences may exacerbate existing biases or lead to convincing yet misleading outputs [@perez2022].
 
-`ccs` is a specialized library developed to provide both the original and an enhanced version of the CSS methodology. Described in the paper "Discovering Latent Knowledge in Language Models Without Supervision" by @burns2022. In addition, we have implemented an approach, called VINC, based on the Contrastive Representation Clustering (CRC) method from the same paper.
+Recent studies indicate that it's possible to extract simulated internal beliefs or 'knowledge' from language model activations [@li2022emergent; @gurnee2023language; @azaria2023internal; @bubeck2023sparks]. While supervised probing techniques can be used for this purpose [@alain2016understanding; @marks2023geometry], they rely on labels that may be compromised by human biases or limitations in human knowledge. In some cases, it's crucial to avoid human labels altogether to allow distinguishing between a model's true knowledge and its representation of human beliefs.
 
-`ccs` serves as a tool for those seeking to investigate the veracity of model output and explore the underlying beliefs embedded within the model. The library offers:
+These considerations have led to the development of unsupervised probing methods, such as Contrast-Consistent Search (CCS) [@burns]. These techniques aim to extract knowledge embedded in language models without relying on ground truth labels [@zou2023representation; @burns2022]. Such approaches offer a promising direction for uncovering the latent knowledge within language models while mitigating the influence of human biases and limitations.
 
-- Multi-GPU Support: Efficient extraction, training, and evaluation through parallel processing.
-- Integration with Huggingface: Easy utilization of models and datasets from a popular source.
-- Active Development and Support: Continuous improvement by a dedicated team of researchers and engineers.
+The aforementioned issues underscore the critical need for tools for researchers to easily train and investiage probes and analyze the internal representations of language models: Our `ccs` library is used to elicit latent knowledge ([elk](`https://docs.google.com/document/d/1WwsnJQstPq91_Yh-Ch2XRL8H_EpsnjrC1dwZXR37PC8/`) [@christiano2021]) from within the activations of a language model. The `ccs` library is developed to provide both the original and an enhanced version of the Contrast-Consistent Search (CCS) method described in the paper "Discovering Latent Knowledge in Language Models Without Supervision" by @burns2022.
+
+Our enhanced version of CCS uses the LBFGS optimizer instead of Adam, which speeds up the training process. Furthermore, it uses learnable Platt scaling parameters to avoid the problem of sign ambiguity from the original implementation.
+
+In addition, we have implemented an approach called VINC (Variance, Invariance, Negative Covariance). VINC is an enhanced method for eliciting latent knowledge from language models. It builds upon the Contrastive Representation Clustering—Top Principal Component (CRC-TPC) [@burns2022] approach and incorporates additional principles. VINC aims to find a direction in activation space that maximizes variance while encouraging negative correlation between statement pairs and paraphrase invariance. The method uses eigendecomposition to optimize a quadratic objective that balances these criteria. VINC can be seen as an alternative to CCS, which takes less time to train. Additional changes and more recent results on VINC and its successor can be found [here](https://blog.eleuther.ai/vincs/).
+
+Finally, we provide a method to train supervised probes using logistic regression, allowing a comparison with unsupervised methods.
+
+`ccs` serves as a tool for researchers to investigate the truthfulness of model outputs and explore the underlying beliefs embedded within the model. The library offers:
+
+- The enhanced or original version of CCS
+- Multi-GPU Support: Efficient extraction, training, and evaluation through parallel processing
+- Integration with Huggingface: Easy utilization of models and datasets from a popular source
+- VINC, an alternative to CCS
+- Training supervised probes with logistic regression for comparisons
 
 For collaboration, discussion, and support, the [Eleuther AI Discord's elk channel](https://discord.com/channels/729741769192767510/1070194752785489991) provides a platform for engaging with others interested in the library or related research projects.
 
 # Acknowledgements
-We would like to thank [EleutherAI](https://www.eleuther.ai/), [SERI MATS](https://www.serimats.org/) for supporting our work and [Long-Term Future Fund (LTFF)](https://funds.effectivealtruism.org/funds/far-future)
+We would like to thank [EleutherAI](https://www.eleuther.ai/), [SERI MATS](https://www.serimats.org/) for supporting our work and [Long-Term Future Fund (LTFF)](https://funds.effectivealtruism.org/funds/far-future).

From 7ae1c46b1c31f6f0f2307a8ed7ab87102020ff50 Mon Sep 17 00:00:00 2001
From: Walter Laurito <lauritowal@yahoo.com>
Date: Sat, 31 Aug 2024 19:36:21 +0200
Subject: [PATCH 76/77] add example project

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 3578d517..6e3464b2 100644
--- a/README.md
+++ b/README.md
@@ -19,7 +19,7 @@ classification tasks, even though the features are trained without labels.
 
 Our code is based on [PyTorch](http://pytorch.org)
 and [Huggingface Transformers](https://huggingface.co/docs/transformers/index). We test the code on Python 3.10 and
-3.11.
+3.11. An example can be found [here](https://colab.research.google.com/drive/1pzcH55aHVXvfF0967hNixReG--gNT473?usp=sharing).
 
 First, create a virtual environment by using e.g. conda:
 

From 129902adcc3c4012b0450e4a63e2765a43e99956 Mon Sep 17 00:00:00 2001
From: Walter Laurito <Lauritowal@yahoo.com>
Date: Sun, 1 Sep 2024 11:01:11 +0200
Subject: [PATCH 77/77] move Kay down

---
 joss/paper.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/joss/paper.md b/joss/paper.md
index 237cff10..01fa8236 100644
--- a/joss/paper.md
+++ b/joss/paper.md
@@ -12,12 +12,12 @@ authors: # sorted by num of commits
     affiliation: "2, 3"
   - name: Nora Belrose
   affiliation: 1 
+  - name: Alex Mallen
+    affiliation: "1, 7"
   - name: Kay Kozaronek
     affiliation: 2
   - name: Fabien Roger
   affiliation: 4
-  - name: Alex Mallen
-    affiliation: "1, 7"
   - name: Christy Koh
     affiliation: 5
   - name: James Chua