diff --git a/_static/coverage/d_37285d613390727b_can_be_used_as_mlp_py.html b/_static/coverage/d_37285d613390727b_can_be_used_as_mlp_py.html
index 1a476d946..d69e4ec4c 100644
--- a/_static/coverage/d_37285d613390727b_can_be_used_as_mlp_py.html
+++ b/_static/coverage/d_37285d613390727b_can_be_used_as_mlp_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_37285d613390727b_gated_mlp_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -166,7 +166,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_37285d613390727b_gated_mlp_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_37285d613390727b_gated_mlp_4bit_py.html b/_static/coverage/d_37285d613390727b_gated_mlp_4bit_py.html
index 36eddcafd..ea1bb3346 100644
--- a/_static/coverage/d_37285d613390727b_gated_mlp_4bit_py.html
+++ b/_static/coverage/d_37285d613390727b_gated_mlp_4bit_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_37285d613390727b_mlp_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -168,7 +168,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_37285d613390727b_mlp_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_37285d613390727b_gated_mlp_py.html b/_static/coverage/d_37285d613390727b_gated_mlp_py.html
index f2c95341f..62782ef92 100644
--- a/_static/coverage/d_37285d613390727b_gated_mlp_py.html
+++ b/_static/coverage/d_37285d613390727b_gated_mlp_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_37285d613390727b_gated_mlp_4bit_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -164,7 +164,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_37285d613390727b_gated_mlp_4bit_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_37285d613390727b_mlp_py.html b/_static/coverage/d_37285d613390727b_mlp_py.html
index f689ef4eb..4c2babbee 100644
--- a/_static/coverage/d_37285d613390727b_mlp_py.html
+++ b/_static/coverage/d_37285d613390727b_mlp_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_37285d613390727b_moe_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -140,7 +140,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_37285d613390727b_moe_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_37285d613390727b_moe_py.html b/_static/coverage/d_37285d613390727b_moe_py.html
index 65dfb1e86..1a5171ee0 100644
--- a/_static/coverage/d_37285d613390727b_moe_py.html
+++ b/_static/coverage/d_37285d613390727b_moe_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_pos_embed_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -204,7 +204,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_pos_embed_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_65d4430f90bfb219_activation_function_factory_py.html b/_static/coverage/d_65d4430f90bfb219_activation_function_factory_py.html
index 2f3abce60..96c147e8c 100644
--- a/_static/coverage/d_65d4430f90bfb219_activation_function_factory_py.html
+++ b/_static/coverage/d_65d4430f90bfb219_activation_function_factory_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_65d4430f90bfb219_mlp_factory_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -128,7 +128,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_65d4430f90bfb219_mlp_factory_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_65d4430f90bfb219_mlp_factory_py.html b/_static/coverage/d_65d4430f90bfb219_mlp_factory_py.html
index e5019d890..f203456b9 100644
--- a/_static/coverage/d_65d4430f90bfb219_mlp_factory_py.html
+++ b/_static/coverage/d_65d4430f90bfb219_mlp_factory_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_head_detector_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -112,7 +112,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_head_detector_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_712808f24eb400fe___init___py.html b/_static/coverage/d_712808f24eb400fe___init___py.html
index dc13b2c3c..d6c43a3f9 100644
--- a/_static/coverage/d_712808f24eb400fe___init___py.html
+++ b/_static/coverage/d_712808f24eb400fe___init___py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7___init___py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -91,7 +91,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7___init___py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_af97b5493da09a14_ActivationCache_py.html b/_static/coverage/d_af97b5493da09a14_ActivationCache_py.html
index e19d362ee..1023238c3 100644
--- a/_static/coverage/d_af97b5493da09a14_ActivationCache_py.html
+++ b/_static/coverage/d_af97b5493da09a14_ActivationCache_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_FactoredMatrix_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -1192,7 +1192,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_FactoredMatrix_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_af97b5493da09a14_FactoredMatrix_py.html b/_static/coverage/d_af97b5493da09a14_FactoredMatrix_py.html
index 68bc0daf1..5fa883efb 100644
--- a/_static/coverage/d_af97b5493da09a14_FactoredMatrix_py.html
+++ b/_static/coverage/d_af97b5493da09a14_FactoredMatrix_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_HookedEncoder_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -365,7 +365,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_HookedEncoder_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_af97b5493da09a14_HookedEncoderDecoder_py.html b/_static/coverage/d_af97b5493da09a14_HookedEncoderDecoder_py.html
index c1264c951..8a9f58a36 100644
--- a/_static/coverage/d_af97b5493da09a14_HookedEncoderDecoder_py.html
+++ b/_static/coverage/d_af97b5493da09a14_HookedEncoderDecoder_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_HookedTransformer_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -507,7 +507,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_HookedTransformer_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_af97b5493da09a14_HookedEncoder_py.html b/_static/coverage/d_af97b5493da09a14_HookedEncoder_py.html
index 87961abdd..47e06c58f 100644
--- a/_static/coverage/d_af97b5493da09a14_HookedEncoder_py.html
+++ b/_static/coverage/d_af97b5493da09a14_HookedEncoder_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_HookedEncoderDecoder_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -461,7 +461,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_HookedEncoderDecoder_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_af97b5493da09a14_HookedTransformerConfig_py.html b/_static/coverage/d_af97b5493da09a14_HookedTransformerConfig_py.html
index ac43a1727..21303717d 100644
--- a/_static/coverage/d_af97b5493da09a14_HookedTransformerConfig_py.html
+++ b/_static/coverage/d_af97b5493da09a14_HookedTransformerConfig_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_SVDInterpreter_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -162,7 +162,7 @@ <h2>
     <p class="pln"><span class="n"><a id="t78" href="#t78">78</a></span><span class="t"><span class="str">            attention</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t79" href="#t79">79</a></span><span class="t"><span class="str">        attn_types (List[str], *optional*): the types of attention to use for</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t80" href="#t80">80</a></span><span class="t"><span class="str">            local attention</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t81" href="#t81">81</a></span><span class="t"><span class="str">        weight_init_mode (str): the initialization mode to use for the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t81" href="#t81">81</a></span><span class="t"><span class="str">        init_mode (str): the initialization mode to use for the</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t82" href="#t82">82</a></span><span class="t"><span class="str">            weights. Only relevant for custom models, ignored for pre-trained.</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t83" href="#t83">83</a></span><span class="t"><span class="str">            We now support 'gpt2', 'xavier_uniform', 'xavier_normal', 'kaiming_uniform',</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t84" href="#t84">84</a></span><span class="t"><span class="str">            'kaiming_normal'. MuP support to come. Defaults to 'gpt2'.</span>&nbsp;</span><span class="r"></span></p>
@@ -184,7 +184,7 @@ <h2>
     <p class="pln"><span class="n"><a id="t100" href="#t100">100</a></span><span class="t"><span class="str">            Used to set sources of randomness (Python, PyTorch and NumPy) and to initialize weights.</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t101" href="#t101">101</a></span><span class="t"><span class="str">            Defaults to None. We recommend setting a seed, so your experiments are reproducible.</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t102" href="#t102">102</a></span><span class="t"><span class="str">        initializer_range (float): The standard deviation of the normal used to</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t103" href="#t103">103</a></span><span class="t"><span class="str">            initialise the weights, initialized to 0.8 / sqrt(d_model). If weight_init_mode is</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t103" href="#t103">103</a></span><span class="t"><span class="str">            initialise the weights, initialized to 0.8 / sqrt(d_model). If init_mode is</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t104" href="#t104">104</a></span><span class="t"><span class="str">            'xavier_uniform' or 'xavier_normal', this value is instead treated as the `gain` parameter for the weight</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t105" href="#t105">105</a></span><span class="t"><span class="str">            initialisation (a constant factor to scale the weights by). Defaults to -1.0, which means not set.</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t106" href="#t106">106</a></span><span class="t"><span class="str">        init_weights (bool): Whether to initialize the weights. Defaults to</span>&nbsp;</span><span class="r"></span></p>
@@ -444,7 +444,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_SVDInterpreter_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_af97b5493da09a14_HookedTransformer_py.html b/_static/coverage/d_af97b5493da09a14_HookedTransformer_py.html
index f4bd2648e..acd621b7d 100644
--- a/_static/coverage/d_af97b5493da09a14_HookedTransformer_py.html
+++ b/_static/coverage/d_af97b5493da09a14_HookedTransformer_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_HookedTransformerConfig_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -1154,1394 +1154,1397 @@ <h2>
     <p class="pln"><span class="n"><a id="t1070" href="#t1070">1070</a></span><span class="t">        <span class="nam">default_prepend_bos</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1071" href="#t1071">1071</a></span><span class="t">        <span class="nam">default_padding_side</span><span class="op">:</span> <span class="nam">Literal</span><span class="op">[</span><span class="str">"left"</span><span class="op">,</span> <span class="str">"right"</span><span class="op">]</span> <span class="op">=</span> <span class="str">"right"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1072" href="#t1072">1072</a></span><span class="t">        <span class="nam">dtype</span><span class="op">=</span><span class="str">"float32"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1073" href="#t1073">1073</a></span><span class="t">        <span class="op">**</span><span class="nam">from_pretrained_kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1074" href="#t1074">1074</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="str">"HookedTransformer"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1075" href="#t1075">1075</a></span><span class="t">        <span class="str">"""Load in a Pretrained Model.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1076" href="#t1076">1076</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1077" href="#t1077">1077</a></span><span class="t"><span class="str">        Load in pretrained model weights to the HookedTransformer format and optionally to do some</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1078" href="#t1078">1078</a></span><span class="t"><span class="str">        processing to make the model easier to interpret. Currently supports loading from most</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1079" href="#t1079">1079</a></span><span class="t"><span class="str">        autoregressive HuggingFace models (``gpt2``, ``neo``, ``gptj``, ``opt``...) and from a range</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1080" href="#t1080">1080</a></span><span class="t"><span class="str">        of toy models and SoLU models trained by Neel Nanda. The full list is available in the docs</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1081" href="#t1081">1081</a></span><span class="t"><span class="str">        under :doc:`model properties&lt;/generated/model_properties_table>`. Also supports loading from</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1082" href="#t1082">1082</a></span><span class="t"><span class="str">        a checkpoint for checkpointed models (currently, models trained by NeelNanda and the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1083" href="#t1083">1083</a></span><span class="t"><span class="str">        stanford-crfm models (using parameters ``checkpoint_index`` and ``checkpoint_value``).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1084" href="#t1084">1084</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1085" href="#t1085">1085</a></span><span class="t"><span class="str">        See :meth:`load_and_process_state_dict` for details on the processing (folding layer norm,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1086" href="#t1086">1086</a></span><span class="t"><span class="str">        centering the unembedding and centering the writing weights).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1087" href="#t1087">1087</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1088" href="#t1088">1088</a></span><span class="t"><span class="str">        Example:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1089" href="#t1089">1089</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1090" href="#t1090">1090</a></span><span class="t"><span class="str">        >>> from transformer_lens import HookedTransformer</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1091" href="#t1091">1091</a></span><span class="t"><span class="str">        >>> model = HookedTransformer.from_pretrained("tiny-stories-1M")</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1092" href="#t1092">1092</a></span><span class="t"><span class="str">        Loaded pretrained model tiny-stories-1M into HookedTransformer</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1093" href="#t1093">1093</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1094" href="#t1094">1094</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1095" href="#t1095">1095</a></span><span class="t"><span class="str">            model_name: The model name - must be an element of</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1096" href="#t1096">1096</a></span><span class="t"><span class="str">                :const:`transformer_lens.loading_from_pretrained.OFFICIAL_MODEL_NAMES` or an alias</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1097" href="#t1097">1097</a></span><span class="t"><span class="str">                of one. The full list of available models can be found in the docs under :doc:`model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1098" href="#t1098">1098</a></span><span class="t"><span class="str">                properties&lt;/generated/model_properties_table>`.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1099" href="#t1099">1099</a></span><span class="t"><span class="str">            fold_ln: Whether to fold in the LayerNorm weights to the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1100" href="#t1100">1100</a></span><span class="t"><span class="str">                subsequent linear layer. This does not change the computation.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1101" href="#t1101">1101</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1102" href="#t1102">1102</a></span><span class="t"><span class="str">                `LayerNorm</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1103" href="#t1103">1103</a></span><span class="t"><span class="str">                &lt;https://wandb.ai/wandb_fc/LayerNorm/reports/Layer-Normalization-in-Pytorch-With-Examples---VmlldzoxMjk5MTk1>`_</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1104" href="#t1104">1104</a></span><span class="t"><span class="str">                is a common regularization technique used in transformers. Unlike BatchNorm, it</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1105" href="#t1105">1105</a></span><span class="t"><span class="str">                cannot be turned off at inference time, as it significantly alters the mathematical</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1106" href="#t1106">1106</a></span><span class="t"><span class="str">                function implemented by the transformer.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1107" href="#t1107">1107</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1108" href="#t1108">1108</a></span><span class="t"><span class="str">                When `fold_ln` is set to True, LayerNorm (with weights :math:`w_{ln}` and</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1109" href="#t1109">1109</a></span><span class="t"><span class="str">                :math:`b_{ln}`) followed by a linear layer (:math:`W + b`) is optimized to</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1110" href="#t1110">1110</a></span><span class="t"><span class="str">                LayerNormPre (just centering &amp; normalizing) followed by a new linear layer with</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1111" href="#t1111">1111</a></span><span class="t"><span class="str">                :math:`W_{eff} = w[:, \text{None}] * W` (element-wise multiplication) and</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1112" href="#t1112">1112</a></span><span class="t"><span class="str">                :math:`b_{eff} = b + b_{ln} @ W`. This transformation is computationally equivalent</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1113" href="#t1113">1113</a></span><span class="t"><span class="str">                and simplifies the model's interpretability. It essentially merges LayerNorm weights</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1114" href="#t1114">1114</a></span><span class="t"><span class="str">                into the subsequent linear layer's weights, which is handled by HookedTransformer</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1115" href="#t1115">1115</a></span><span class="t"><span class="str">                when loading pre-trained weights. Set `fold_ln` to False when loading a state dict</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1116" href="#t1116">1116</a></span><span class="t"><span class="str">                if you wish to turn this off.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1117" href="#t1117">1117</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1118" href="#t1118">1118</a></span><span class="t"><span class="str">                Mathematically, LayerNorm is defined as follows:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1119" href="#t1119">1119</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1120" href="#t1120">1120</a></span><span class="t"><span class="str">                .. math::</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1121" href="#t1121">1121</a></span><span class="t"><span class="str">                    x_1 &amp;= x_0 - \\text{mean}(x_0)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1122" href="#t1122">1122</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1123" href="#t1123">1123</a></span><span class="t"><span class="str">                    x_2 &amp;= \\frac{x_1}{\\sqrt{\\text{mean}(x_1^2)}}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1124" href="#t1124">1124</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1125" href="#t1125">1125</a></span><span class="t"><span class="str">                    x_3 &amp;= x_2 \\cdot w</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1126" href="#t1126">1126</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1127" href="#t1127">1127</a></span><span class="t"><span class="str">                    x_4 &amp;= x_3 + b</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1128" href="#t1128">1128</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1129" href="#t1129">1129</a></span><span class="t"><span class="str">                For further details, refer to `this document</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1130" href="#t1130">1130</a></span><span class="t"><span class="str">                &lt;https://transformer-circuits.pub/2021/framework/index.html#:~:text=Handling%20Layer%20Normalization>`_.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1131" href="#t1131">1131</a></span><span class="t"><span class="str">            center_writing_weights: Whether to center weights</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1132" href="#t1132">1132</a></span><span class="t"><span class="str">                writing to the residual stream (ie set mean to be zero). Due to LayerNorm this</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1133" href="#t1133">1133</a></span><span class="t"><span class="str">                doesn't change the computation.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1134" href="#t1134">1134</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1135" href="#t1135">1135</a></span><span class="t"><span class="str">                A related idea to folding layernorm (``fold_ln``) - *every* component reading an</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1136" href="#t1136">1136</a></span><span class="t"><span class="str">                input from the residual stream is preceded by a LayerNorm, which means that the mean</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1137" href="#t1137">1137</a></span><span class="t"><span class="str">                of a residual stream vector (ie the component in the direction of all ones) never</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1138" href="#t1138">1138</a></span><span class="t"><span class="str">                matters. This means we can remove the all ones component of weights and biases whose</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1139" href="#t1139">1139</a></span><span class="t"><span class="str">                output *writes* to the residual stream. Mathematically, ``W_writing -=</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1140" href="#t1140">1140</a></span><span class="t"><span class="str">                W_writing.mean(dim=1, keepdim=True)``.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1141" href="#t1141">1141</a></span><span class="t"><span class="str">            center_unembed: Whether to center W_U (ie set mean</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1142" href="#t1142">1142</a></span><span class="t"><span class="str">                to be zero). Softmax is translation invariant so this doesn't affect log probs or</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1143" href="#t1143">1143</a></span><span class="t"><span class="str">                loss, but does change logits.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1144" href="#t1144">1144</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1145" href="#t1145">1145</a></span><span class="t"><span class="str">                The logits are fed into a softmax. Softmax is translation invariant (eg, adding 1 to</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1146" href="#t1146">1146</a></span><span class="t"><span class="str">                every logit doesn't change the output), so we can simplify things by setting the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1147" href="#t1147">1147</a></span><span class="t"><span class="str">                mean of the logits to be zero. This is equivalent to setting the mean of every</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1148" href="#t1148">1148</a></span><span class="t"><span class="str">                output vector of ``W_U`` to zero. In code, ``W_U -= W_U.mean(dim=-1,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1149" href="#t1149">1149</a></span><span class="t"><span class="str">                keepdim=True)``.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1150" href="#t1150">1150</a></span><span class="t"><span class="str">            refactor_factored_attn_matrices: Whether to convert the factored</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1151" href="#t1151">1151</a></span><span class="t"><span class="str">                matrices (W_Q &amp; W_K, and W_O &amp; W_V) to be "even". Defaults to False</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1152" href="#t1152">1152</a></span><span class="t"><span class="str">            checkpoint_index: If loading from a checkpoint, the index of</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1153" href="#t1153">1153</a></span><span class="t"><span class="str">                the checkpoint to load.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1154" href="#t1154">1154</a></span><span class="t"><span class="str">            checkpoint_value: If loading from a checkpoint, the value of</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1155" href="#t1155">1155</a></span><span class="t"><span class="str">                the checkpoint to load, ie the step or token number (each model has checkpoints</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1156" href="#t1156">1156</a></span><span class="t"><span class="str">                labelled with exactly one of these). E.g. ``1000`` for a checkpoint taken at step</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1157" href="#t1157">1157</a></span><span class="t"><span class="str">                1000 or after 1000 tokens. If `checkpoint_index` is also specified, this will be</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1158" href="#t1158">1158</a></span><span class="t"><span class="str">                ignored.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1159" href="#t1159">1159</a></span><span class="t"><span class="str">            hf_model: If you have already loaded in the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1160" href="#t1160">1160</a></span><span class="t"><span class="str">                HuggingFace model, you can pass it in here rather than needing to recreate the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1161" href="#t1161">1161</a></span><span class="t"><span class="str">                object. Defaults to None.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1162" href="#t1162">1162</a></span><span class="t"><span class="str">            device: The device to load the model onto. By</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1163" href="#t1163">1163</a></span><span class="t"><span class="str">                default will load to CUDA if available, else CPU.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1164" href="#t1164">1164</a></span><span class="t"><span class="str">            n_devices: The number of devices to split the model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1165" href="#t1165">1165</a></span><span class="t"><span class="str">                across. Defaults to 1. If greater than 1, `device` must be cuda.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1166" href="#t1166">1166</a></span><span class="t"><span class="str">            tokenizer: The tokenizer to use for the model. If not</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1167" href="#t1167">1167</a></span><span class="t"><span class="str">                provided, it is inferred from cfg.tokenizer_name or initialized to None. If None,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1168" href="#t1168">1168</a></span><span class="t"><span class="str">                then the model cannot be passed strings, and d_vocab must be explicitly set.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1169" href="#t1169">1169</a></span><span class="t"><span class="str">            move_to_device: Whether to move the model to the device specified in</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1170" href="#t1170">1170</a></span><span class="t"><span class="str">                cfg. device. Must be true if `n_devices` in the config is greater than 1, since the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1171" href="#t1171">1171</a></span><span class="t"><span class="str">                model's layers will be split across multiple devices.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1172" href="#t1172">1172</a></span><span class="t"><span class="str">            fold_value_biases: Each attention head has a value bias. Values are averaged to create</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1173" href="#t1173">1173</a></span><span class="t"><span class="str">                mixed values (``z``), weighted by the attention pattern, but as the bias is</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1174" href="#t1174">1174</a></span><span class="t"><span class="str">                constant, its contribution to ``z`` is exactly the same. The output of a head is ``z</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1175" href="#t1175">1175</a></span><span class="t"><span class="str">                @ W_O``, and so the value bias just linearly adds to the output of the head. This</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1176" href="#t1176">1176</a></span><span class="t"><span class="str">                means that the value bias of a head has nothing to do with the head, and is just a</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1177" href="#t1177">1177</a></span><span class="t"><span class="str">                constant added to the attention layer outputs. We can take the sum across these and</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1178" href="#t1178">1178</a></span><span class="t"><span class="str">                b_O to get an "effective bias" for the layer. In code, we set ``b_V=0``. and ``b_O =</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1179" href="#t1179">1179</a></span><span class="t"><span class="str">                (b_V @ W_O).sum(dim=0) + b_O``.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1180" href="#t1180">1180</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1181" href="#t1181">1181</a></span><span class="t"><span class="str">                The technical derivation of this is as follows. ``v = residual @ W_V[h] +</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1182" href="#t1182">1182</a></span><span class="t"><span class="str">                broadcast_b_V[h]`` for each head ``h`` (where ``b_V`` is broadcast up from shape</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1183" href="#t1183">1183</a></span><span class="t"><span class="str">                ``d_head`` to shape ``[position, d_head]``). And ``z = pattern[h] @ v = pattern[h] @</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1184" href="#t1184">1184</a></span><span class="t"><span class="str">                residual @ W_V[h] + pattern[h] @ broadcast_b_V[h]``. Because ``pattern[h]`` is</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1185" href="#t1185">1185</a></span><span class="t"><span class="str">                ``[destination_position, source_position]`` and ``broadcast_b_V`` is constant along</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1186" href="#t1186">1186</a></span><span class="t"><span class="str">                the ``(source_)position`` dimension, we're basically just multiplying it by the sum</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1187" href="#t1187">1187</a></span><span class="t"><span class="str">                of the pattern across the ``source_position`` dimension, which is just ``1``. So it</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1188" href="#t1188">1188</a></span><span class="t"><span class="str">                remains exactly the same, and so is just broadcast across the destination positions.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1189" href="#t1189">1189</a></span><span class="t"><span class="str">            default_prepend_bos: Default behavior of whether to prepend the BOS</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1190" href="#t1190">1190</a></span><span class="t"><span class="str">                token when the methods of HookedTransformer process input text to tokenize (only</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1191" href="#t1191">1191</a></span><span class="t"><span class="str">                when input is a string). Defaults to True - even for models not explicitly trained</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1192" href="#t1192">1192</a></span><span class="t"><span class="str">                with this, heads often use the first position as a resting position and accordingly</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1193" href="#t1193">1193</a></span><span class="t"><span class="str">                lose information from the first token, so this empirically seems to give better</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1194" href="#t1194">1194</a></span><span class="t"><span class="str">                results. To change the default behavior to False, pass in default_prepend_bos=False.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1195" href="#t1195">1195</a></span><span class="t"><span class="str">                Note that you can also locally override the default behavior by passing in</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1196" href="#t1196">1196</a></span><span class="t"><span class="str">                prepend_bos=True/False when you call a method that processes the input string.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1197" href="#t1197">1197</a></span><span class="t"><span class="str">            from_pretrained_kwargs: Any other optional argument passed to</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1198" href="#t1198">1198</a></span><span class="t"><span class="str">                HuggingFace's from_pretrained (e.g. "cache_dir" or "torch_dtype"). Also passed to</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1199" href="#t1199">1199</a></span><span class="t"><span class="str">                other HuggingFace functions when compatible. For some models or arguments it doesn't</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1200" href="#t1200">1200</a></span><span class="t"><span class="str">                work, especially for models that are not internally loaded with HuggingFace's</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1201" href="#t1201">1201</a></span><span class="t"><span class="str">                from_pretrained (e.g. SoLU models).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1202" href="#t1202">1202</a></span><span class="t"><span class="str">            dtype: What data type to load the model in (also sets the dtype of</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1203" href="#t1203">1203</a></span><span class="t"><span class="str">                the HuggingFace model). Set to bfloat16 or float16 if you get out of memory errors when loading</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1204" href="#t1204">1204</a></span><span class="t"><span class="str">                the model.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1205" href="#t1205">1205</a></span><span class="t"><span class="str">            default_padding_side: Which side to pad on when tokenizing. Defaults to</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1206" href="#t1206">1206</a></span><span class="t"><span class="str">                "right".</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1207" href="#t1207">1207</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1208" href="#t1208">1208</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1209" href="#t1209">1209</a></span><span class="t">        <span class="key">assert</span> <span class="key">not</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1210" href="#t1210">1210</a></span><span class="t">            <span class="nam">from_pretrained_kwargs</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"load_in_8bit"</span><span class="op">,</span> <span class="key">False</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1211" href="#t1211">1211</a></span><span class="t">            <span class="key">or</span> <span class="nam">from_pretrained_kwargs</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"load_in_4bit"</span><span class="op">,</span> <span class="key">False</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1212" href="#t1212">1212</a></span><span class="t">        <span class="op">)</span><span class="op">,</span> <span class="str">"Quantization not supported"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1213" href="#t1213">1213</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1214" href="#t1214">1214</a></span><span class="t">        <span class="key">if</span> <span class="nam">hf_model</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1214&#x202F;&#x219B;&#x202F;1215</span><span class="annotate long">line 1214 didn't jump to line 1215, because the condition on line 1214 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1215" href="#t1215">1215</a></span><span class="t">            <span class="nam">hf_cfg</span> <span class="op">=</span> <span class="nam">hf_model</span><span class="op">.</span><span class="nam">config</span><span class="op">.</span><span class="nam">to_dict</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1216" href="#t1216">1216</a></span><span class="t">            <span class="nam">qc</span> <span class="op">=</span> <span class="nam">hf_cfg</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"quantization_config"</span><span class="op">,</span> <span class="op">{</span><span class="op">}</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1217" href="#t1217">1217</a></span><span class="t">            <span class="nam">load_in_4bit</span> <span class="op">=</span> <span class="nam">qc</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"load_in_4bit"</span><span class="op">,</span> <span class="key">False</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1218" href="#t1218">1218</a></span><span class="t">            <span class="nam">load_in_8bit</span> <span class="op">=</span> <span class="nam">qc</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"load_in_8bit"</span><span class="op">,</span> <span class="key">False</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1219" href="#t1219">1219</a></span><span class="t">            <span class="nam">quant_method</span> <span class="op">=</span> <span class="nam">qc</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"quant_method"</span><span class="op">,</span> <span class="str">""</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1220" href="#t1220">1220</a></span><span class="t">            <span class="key">assert</span> <span class="key">not</span> <span class="nam">load_in_8bit</span><span class="op">,</span> <span class="str">"8-bit quantization is not supported"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1221" href="#t1221">1221</a></span><span class="t">            <span class="key">assert</span> <span class="key">not</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1222" href="#t1222">1222</a></span><span class="t">                <span class="nam">load_in_4bit</span> <span class="key">and</span> <span class="op">(</span><span class="nam">version</span><span class="op">.</span><span class="nam">parse</span><span class="op">(</span><span class="nam">torch</span><span class="op">.</span><span class="nam">__version__</span><span class="op">)</span> <span class="op">&lt;</span> <span class="nam">version</span><span class="op">.</span><span class="nam">parse</span><span class="op">(</span><span class="str">"2.1.1"</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1223" href="#t1223">1223</a></span><span class="t">            <span class="op">)</span><span class="op">,</span> <span class="str">"Quantization is only supported for torch versions >= 2.1.1"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1224" href="#t1224">1224</a></span><span class="t">            <span class="key">assert</span> <span class="key">not</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1225" href="#t1225">1225</a></span><span class="t">                <span class="nam">load_in_4bit</span> <span class="key">and</span> <span class="op">(</span><span class="str">"llama"</span> <span class="key">not</span> <span class="key">in</span> <span class="nam">model_name</span><span class="op">.</span><span class="nam">lower</span><span class="op">(</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1226" href="#t1226">1226</a></span><span class="t">            <span class="op">)</span><span class="op">,</span> <span class="str">"Quantization is only supported for Llama models"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1227" href="#t1227">1227</a></span><span class="t">            <span class="key">if</span> <span class="nam">load_in_4bit</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1228" href="#t1228">1228</a></span><span class="t">                <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1229" href="#t1229">1229</a></span><span class="t">                    <span class="nam">qc</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"quant_method"</span><span class="op">,</span> <span class="str">""</span><span class="op">)</span> <span class="op">==</span> <span class="str">"bitsandbytes"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1230" href="#t1230">1230</a></span><span class="t">                <span class="op">)</span><span class="op">,</span> <span class="str">"Only bitsandbytes quantization is supported"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1231" href="#t1231">1231</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1232" href="#t1232">1232</a></span><span class="t">            <span class="nam">hf_cfg</span> <span class="op">=</span> <span class="op">{</span><span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1233" href="#t1233">1233</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1234" href="#t1234">1234</a></span><span class="t">        <span class="key">if</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">dtype</span><span class="op">,</span> <span class="nam">str</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1235" href="#t1235">1235</a></span><span class="t">            <span class="com"># Convert from string to a torch dtype</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1236" href="#t1236">1236</a></span><span class="t">            <span class="nam">dtype</span> <span class="op">=</span> <span class="nam">DTYPE_FROM_STRING</span><span class="op">[</span><span class="nam">dtype</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1237" href="#t1237">1237</a></span><span class="t">        <span class="key">if</span> <span class="str">"torch_dtype"</span> <span class="key">in</span> <span class="nam">from_pretrained_kwargs</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1237&#x202F;&#x219B;&#x202F;1240</span><span class="annotate long">line 1237 didn't jump to line 1240, because the condition on line 1237 was never true</span></span></p>
-    <p class="pln"><span class="n"><a id="t1238" href="#t1238">1238</a></span><span class="t">            <span class="com"># For backwards compatibility with the previous way to do low precision loading</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1239" href="#t1239">1239</a></span><span class="t">            <span class="com"># This should maybe check the user did not explicitly set dtype *and* torch_dtype</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1240" href="#t1240">1240</a></span><span class="t">            <span class="nam">dtype</span> <span class="op">=</span> <span class="nam">from_pretrained_kwargs</span><span class="op">[</span><span class="str">"torch_dtype"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1241" href="#t1241">1241</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1242" href="#t1242">1242</a></span><span class="t">        <span class="key">if</span> <span class="op">(</span>&nbsp;</span><span class="r"><span class="annotate short">1242&#x202F;&#x219B;&#x202F;1246</span><span class="annotate long">line 1242 didn't jump to line 1246, because the condition on line 1242 was never true</span></span></p>
-    <p class="pln"><span class="n"><a id="t1243" href="#t1243">1243</a></span><span class="t">            <span class="op">(</span><span class="nam">from_pretrained_kwargs</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"torch_dtype"</span><span class="op">,</span> <span class="key">None</span><span class="op">)</span> <span class="op">==</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">float16</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1244" href="#t1244">1244</a></span><span class="t">            <span class="key">or</span> <span class="nam">dtype</span> <span class="op">==</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">float16</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1245" href="#t1245">1245</a></span><span class="t">        <span class="op">)</span> <span class="key">and</span> <span class="nam">device</span> <span class="key">in</span> <span class="op">[</span><span class="str">"cpu"</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1246" href="#t1246">1246</a></span><span class="t">            <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span><span class="str">"float16 models may not work on CPU. Consider using a GPU or bfloat16."</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1247" href="#t1247">1247</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1248" href="#t1248">1248</a></span><span class="t">        <span class="com"># Get the model name used in HuggingFace, rather than the alias.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1249" href="#t1249">1249</a></span><span class="t">        <span class="nam">official_model_name</span> <span class="op">=</span> <span class="nam">loading</span><span class="op">.</span><span class="nam">get_official_model_name</span><span class="op">(</span><span class="nam">model_name</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1250" href="#t1250">1250</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1251" href="#t1251">1251</a></span><span class="t">        <span class="com"># Load the config into an HookedTransformerConfig object. If loading from a</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1252" href="#t1252">1252</a></span><span class="t">        <span class="com"># checkpoint, the config object will contain the information about the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1253" href="#t1253">1253</a></span><span class="t">        <span class="com"># checkpoint</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1254" href="#t1254">1254</a></span><span class="t">        <span class="nam">cfg</span> <span class="op">=</span> <span class="nam">loading</span><span class="op">.</span><span class="nam">get_pretrained_model_config</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1255" href="#t1255">1255</a></span><span class="t">            <span class="nam">official_model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1256" href="#t1256">1256</a></span><span class="t">            <span class="nam">hf_cfg</span><span class="op">=</span><span class="nam">hf_cfg</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1257" href="#t1257">1257</a></span><span class="t">            <span class="nam">checkpoint_index</span><span class="op">=</span><span class="nam">checkpoint_index</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1258" href="#t1258">1258</a></span><span class="t">            <span class="nam">checkpoint_value</span><span class="op">=</span><span class="nam">checkpoint_value</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1259" href="#t1259">1259</a></span><span class="t">            <span class="nam">fold_ln</span><span class="op">=</span><span class="nam">fold_ln</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1260" href="#t1260">1260</a></span><span class="t">            <span class="nam">device</span><span class="op">=</span><span class="nam">device</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1261" href="#t1261">1261</a></span><span class="t">            <span class="nam">n_devices</span><span class="op">=</span><span class="nam">n_devices</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1262" href="#t1262">1262</a></span><span class="t">            <span class="nam">default_prepend_bos</span><span class="op">=</span><span class="nam">default_prepend_bos</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1263" href="#t1263">1263</a></span><span class="t">            <span class="nam">dtype</span><span class="op">=</span><span class="nam">dtype</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1264" href="#t1264">1264</a></span><span class="t">            <span class="op">**</span><span class="nam">from_pretrained_kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1265" href="#t1265">1265</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1266" href="#t1266">1266</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1267" href="#t1267">1267</a></span><span class="t">        <span class="key">if</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">positional_embedding_type</span> <span class="op">==</span> <span class="str">"shortformer"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1268" href="#t1268">1268</a></span><span class="t">            <span class="key">if</span> <span class="nam">fold_ln</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1269" href="#t1269">1269</a></span><span class="t">                <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1270" href="#t1270">1270</a></span><span class="t">                    <span class="str">"You tried to specify fold_ln=True for a shortformer model, but this can't be done! Setting fold_"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1271" href="#t1271">1271</a></span><span class="t">                    <span class="str">"ln=False instead."</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1272" href="#t1272">1272</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1273" href="#t1273">1273</a></span><span class="t">                <span class="nam">fold_ln</span> <span class="op">=</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1274" href="#t1274">1274</a></span><span class="t">            <span class="key">if</span> <span class="nam">center_unembed</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1275" href="#t1275">1275</a></span><span class="t">                <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1276" href="#t1276">1276</a></span><span class="t">                    <span class="str">"You tried to specify center_unembed=True for a shortformer model, but this can't be done! "</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1277" href="#t1277">1277</a></span><span class="t">                    <span class="str">"Setting center_unembed=False instead."</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1278" href="#t1278">1278</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1279" href="#t1279">1279</a></span><span class="t">                <span class="nam">center_unembed</span> <span class="op">=</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1280" href="#t1280">1280</a></span><span class="t">            <span class="key">if</span> <span class="nam">center_writing_weights</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1281" href="#t1281">1281</a></span><span class="t">                <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1282" href="#t1282">1282</a></span><span class="t">                    <span class="str">"You tried to specify center_writing_weights=True for a shortformer model, but this can't be done! "</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1283" href="#t1283">1283</a></span><span class="t">                    <span class="str">"Setting center_writing_weights=False instead."</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1284" href="#t1284">1284</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1285" href="#t1285">1285</a></span><span class="t">                <span class="nam">center_writing_weights</span> <span class="op">=</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1286" href="#t1286">1286</a></span><span class="t">        <span class="key">if</span> <span class="nam">center_unembed</span> <span class="key">and</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">output_logits_soft_cap</span> <span class="op">></span> <span class="num">0.0</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1286&#x202F;&#x219B;&#x202F;1287</span><span class="annotate long">line 1286 didn't jump to line 1287, because the condition on line 1286 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1287" href="#t1287">1287</a></span><span class="t">            <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1288" href="#t1288">1288</a></span><span class="t">                <span class="str">"You tried to specify center_unembed=True for a model using logit softcap, but this can't be done! Softcapping is not invariant upon adding a constant"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1289" href="#t1289">1289</a></span><span class="t">                <span class="str">"Setting center_unembed=False instead."</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1290" href="#t1290">1290</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1291" href="#t1291">1291</a></span><span class="t">            <span class="nam">center_unembed</span> <span class="op">=</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1292" href="#t1292">1292</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1293" href="#t1293">1293</a></span><span class="t">        <span class="com"># Get the state dict of the model (ie a mapping of parameter names to tensors), processed to</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1294" href="#t1294">1294</a></span><span class="t">        <span class="com"># match the HookedTransformer parameter names.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1295" href="#t1295">1295</a></span><span class="t">        <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">loading</span><span class="op">.</span><span class="nam">get_pretrained_state_dict</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1296" href="#t1296">1296</a></span><span class="t">            <span class="nam">official_model_name</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">,</span> <span class="nam">hf_model</span><span class="op">,</span> <span class="nam">dtype</span><span class="op">=</span><span class="nam">dtype</span><span class="op">,</span> <span class="op">**</span><span class="nam">from_pretrained_kwargs</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1297" href="#t1297">1297</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1298" href="#t1298">1298</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1299" href="#t1299">1299</a></span><span class="t">        <span class="com"># Create the HookedTransformer object</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1300" href="#t1300">1300</a></span><span class="t">        <span class="nam">model</span> <span class="op">=</span> <span class="nam">cls</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1301" href="#t1301">1301</a></span><span class="t">            <span class="nam">cfg</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1302" href="#t1302">1302</a></span><span class="t">            <span class="nam">tokenizer</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1303" href="#t1303">1303</a></span><span class="t">            <span class="nam">move_to_device</span><span class="op">=</span><span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1304" href="#t1304">1304</a></span><span class="t">            <span class="nam">default_padding_side</span><span class="op">=</span><span class="nam">default_padding_side</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1305" href="#t1305">1305</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1306" href="#t1306">1306</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1307" href="#t1307">1307</a></span><span class="t">        <span class="nam">model</span><span class="op">.</span><span class="nam">load_and_process_state_dict</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1308" href="#t1308">1308</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1309" href="#t1309">1309</a></span><span class="t">            <span class="nam">fold_ln</span><span class="op">=</span><span class="nam">fold_ln</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1310" href="#t1310">1310</a></span><span class="t">            <span class="nam">center_writing_weights</span><span class="op">=</span><span class="nam">center_writing_weights</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1311" href="#t1311">1311</a></span><span class="t">            <span class="nam">center_unembed</span><span class="op">=</span><span class="nam">center_unembed</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1312" href="#t1312">1312</a></span><span class="t">            <span class="nam">fold_value_biases</span><span class="op">=</span><span class="nam">fold_value_biases</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1313" href="#t1313">1313</a></span><span class="t">            <span class="nam">refactor_factored_attn_matrices</span><span class="op">=</span><span class="nam">refactor_factored_attn_matrices</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1314" href="#t1314">1314</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1315" href="#t1315">1315</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1316" href="#t1316">1316</a></span><span class="t">        <span class="key">if</span> <span class="nam">move_to_device</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1316&#x202F;&#x219B;&#x202F;1319</span><span class="annotate long">line 1316 didn't jump to line 1319, because the condition on line 1316 was never false</span></span></p>
-    <p class="run"><span class="n"><a id="t1317" href="#t1317">1317</a></span><span class="t">            <span class="nam">model</span><span class="op">.</span><span class="nam">move_model_modules_to_device</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1073" href="#t1073">1073</a></span><span class="t">        <span class="nam">first_n_layers</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1074" href="#t1074">1074</a></span><span class="t">        <span class="op">**</span><span class="nam">from_pretrained_kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1075" href="#t1075">1075</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="str">"HookedTransformer"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1076" href="#t1076">1076</a></span><span class="t">        <span class="str">"""Load in a Pretrained Model.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1077" href="#t1077">1077</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1078" href="#t1078">1078</a></span><span class="t"><span class="str">        Load in pretrained model weights to the HookedTransformer format and optionally to do some</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1079" href="#t1079">1079</a></span><span class="t"><span class="str">        processing to make the model easier to interpret. Currently supports loading from most</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1080" href="#t1080">1080</a></span><span class="t"><span class="str">        autoregressive HuggingFace models (``gpt2``, ``neo``, ``gptj``, ``opt``...) and from a range</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1081" href="#t1081">1081</a></span><span class="t"><span class="str">        of toy models and SoLU models trained by Neel Nanda. The full list is available in the docs</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1082" href="#t1082">1082</a></span><span class="t"><span class="str">        under :doc:`model properties&lt;/generated/model_properties_table>`. Also supports loading from</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1083" href="#t1083">1083</a></span><span class="t"><span class="str">        a checkpoint for checkpointed models (currently, models trained by NeelNanda and the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1084" href="#t1084">1084</a></span><span class="t"><span class="str">        stanford-crfm models (using parameters ``checkpoint_index`` and ``checkpoint_value``).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1085" href="#t1085">1085</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1086" href="#t1086">1086</a></span><span class="t"><span class="str">        See :meth:`load_and_process_state_dict` for details on the processing (folding layer norm,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1087" href="#t1087">1087</a></span><span class="t"><span class="str">        centering the unembedding and centering the writing weights).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1088" href="#t1088">1088</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1089" href="#t1089">1089</a></span><span class="t"><span class="str">        Example:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1090" href="#t1090">1090</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1091" href="#t1091">1091</a></span><span class="t"><span class="str">        >>> from transformer_lens import HookedTransformer</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1092" href="#t1092">1092</a></span><span class="t"><span class="str">        >>> model = HookedTransformer.from_pretrained("tiny-stories-1M")</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1093" href="#t1093">1093</a></span><span class="t"><span class="str">        Loaded pretrained model tiny-stories-1M into HookedTransformer</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1094" href="#t1094">1094</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1095" href="#t1095">1095</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1096" href="#t1096">1096</a></span><span class="t"><span class="str">            model_name: The model name - must be an element of</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1097" href="#t1097">1097</a></span><span class="t"><span class="str">                :const:`transformer_lens.loading_from_pretrained.OFFICIAL_MODEL_NAMES` or an alias</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1098" href="#t1098">1098</a></span><span class="t"><span class="str">                of one. The full list of available models can be found in the docs under :doc:`model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1099" href="#t1099">1099</a></span><span class="t"><span class="str">                properties&lt;/generated/model_properties_table>`.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1100" href="#t1100">1100</a></span><span class="t"><span class="str">            fold_ln: Whether to fold in the LayerNorm weights to the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1101" href="#t1101">1101</a></span><span class="t"><span class="str">                subsequent linear layer. This does not change the computation.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1102" href="#t1102">1102</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1103" href="#t1103">1103</a></span><span class="t"><span class="str">                `LayerNorm</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1104" href="#t1104">1104</a></span><span class="t"><span class="str">                &lt;https://wandb.ai/wandb_fc/LayerNorm/reports/Layer-Normalization-in-Pytorch-With-Examples---VmlldzoxMjk5MTk1>`_</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1105" href="#t1105">1105</a></span><span class="t"><span class="str">                is a common regularization technique used in transformers. Unlike BatchNorm, it</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1106" href="#t1106">1106</a></span><span class="t"><span class="str">                cannot be turned off at inference time, as it significantly alters the mathematical</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1107" href="#t1107">1107</a></span><span class="t"><span class="str">                function implemented by the transformer.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1108" href="#t1108">1108</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1109" href="#t1109">1109</a></span><span class="t"><span class="str">                When `fold_ln` is set to True, LayerNorm (with weights :math:`w_{ln}` and</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1110" href="#t1110">1110</a></span><span class="t"><span class="str">                :math:`b_{ln}`) followed by a linear layer (:math:`W + b`) is optimized to</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1111" href="#t1111">1111</a></span><span class="t"><span class="str">                LayerNormPre (just centering &amp; normalizing) followed by a new linear layer with</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1112" href="#t1112">1112</a></span><span class="t"><span class="str">                :math:`W_{eff} = w[:, \text{None}] * W` (element-wise multiplication) and</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1113" href="#t1113">1113</a></span><span class="t"><span class="str">                :math:`b_{eff} = b + b_{ln} @ W`. This transformation is computationally equivalent</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1114" href="#t1114">1114</a></span><span class="t"><span class="str">                and simplifies the model's interpretability. It essentially merges LayerNorm weights</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1115" href="#t1115">1115</a></span><span class="t"><span class="str">                into the subsequent linear layer's weights, which is handled by HookedTransformer</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1116" href="#t1116">1116</a></span><span class="t"><span class="str">                when loading pre-trained weights. Set `fold_ln` to False when loading a state dict</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1117" href="#t1117">1117</a></span><span class="t"><span class="str">                if you wish to turn this off.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1118" href="#t1118">1118</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1119" href="#t1119">1119</a></span><span class="t"><span class="str">                Mathematically, LayerNorm is defined as follows:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1120" href="#t1120">1120</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1121" href="#t1121">1121</a></span><span class="t"><span class="str">                .. math::</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1122" href="#t1122">1122</a></span><span class="t"><span class="str">                    x_1 &amp;= x_0 - \\text{mean}(x_0)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1123" href="#t1123">1123</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1124" href="#t1124">1124</a></span><span class="t"><span class="str">                    x_2 &amp;= \\frac{x_1}{\\sqrt{\\text{mean}(x_1^2)}}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1125" href="#t1125">1125</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1126" href="#t1126">1126</a></span><span class="t"><span class="str">                    x_3 &amp;= x_2 \\cdot w</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1127" href="#t1127">1127</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1128" href="#t1128">1128</a></span><span class="t"><span class="str">                    x_4 &amp;= x_3 + b</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1129" href="#t1129">1129</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1130" href="#t1130">1130</a></span><span class="t"><span class="str">                For further details, refer to `this document</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1131" href="#t1131">1131</a></span><span class="t"><span class="str">                &lt;https://transformer-circuits.pub/2021/framework/index.html#:~:text=Handling%20Layer%20Normalization>`_.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1132" href="#t1132">1132</a></span><span class="t"><span class="str">            center_writing_weights: Whether to center weights</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1133" href="#t1133">1133</a></span><span class="t"><span class="str">                writing to the residual stream (ie set mean to be zero). Due to LayerNorm this</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1134" href="#t1134">1134</a></span><span class="t"><span class="str">                doesn't change the computation.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1135" href="#t1135">1135</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1136" href="#t1136">1136</a></span><span class="t"><span class="str">                A related idea to folding layernorm (``fold_ln``) - *every* component reading an</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1137" href="#t1137">1137</a></span><span class="t"><span class="str">                input from the residual stream is preceded by a LayerNorm, which means that the mean</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1138" href="#t1138">1138</a></span><span class="t"><span class="str">                of a residual stream vector (ie the component in the direction of all ones) never</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1139" href="#t1139">1139</a></span><span class="t"><span class="str">                matters. This means we can remove the all ones component of weights and biases whose</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1140" href="#t1140">1140</a></span><span class="t"><span class="str">                output *writes* to the residual stream. Mathematically, ``W_writing -=</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1141" href="#t1141">1141</a></span><span class="t"><span class="str">                W_writing.mean(dim=1, keepdim=True)``.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1142" href="#t1142">1142</a></span><span class="t"><span class="str">            center_unembed: Whether to center W_U (ie set mean</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1143" href="#t1143">1143</a></span><span class="t"><span class="str">                to be zero). Softmax is translation invariant so this doesn't affect log probs or</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1144" href="#t1144">1144</a></span><span class="t"><span class="str">                loss, but does change logits.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1145" href="#t1145">1145</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1146" href="#t1146">1146</a></span><span class="t"><span class="str">                The logits are fed into a softmax. Softmax is translation invariant (eg, adding 1 to</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1147" href="#t1147">1147</a></span><span class="t"><span class="str">                every logit doesn't change the output), so we can simplify things by setting the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1148" href="#t1148">1148</a></span><span class="t"><span class="str">                mean of the logits to be zero. This is equivalent to setting the mean of every</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1149" href="#t1149">1149</a></span><span class="t"><span class="str">                output vector of ``W_U`` to zero. In code, ``W_U -= W_U.mean(dim=-1,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1150" href="#t1150">1150</a></span><span class="t"><span class="str">                keepdim=True)``.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1151" href="#t1151">1151</a></span><span class="t"><span class="str">            refactor_factored_attn_matrices: Whether to convert the factored</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1152" href="#t1152">1152</a></span><span class="t"><span class="str">                matrices (W_Q &amp; W_K, and W_O &amp; W_V) to be "even". Defaults to False</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1153" href="#t1153">1153</a></span><span class="t"><span class="str">            checkpoint_index: If loading from a checkpoint, the index of</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1154" href="#t1154">1154</a></span><span class="t"><span class="str">                the checkpoint to load.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1155" href="#t1155">1155</a></span><span class="t"><span class="str">            checkpoint_value: If loading from a checkpoint, the value of</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1156" href="#t1156">1156</a></span><span class="t"><span class="str">                the checkpoint to load, ie the step or token number (each model has checkpoints</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1157" href="#t1157">1157</a></span><span class="t"><span class="str">                labelled with exactly one of these). E.g. ``1000`` for a checkpoint taken at step</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1158" href="#t1158">1158</a></span><span class="t"><span class="str">                1000 or after 1000 tokens. If `checkpoint_index` is also specified, this will be</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1159" href="#t1159">1159</a></span><span class="t"><span class="str">                ignored.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1160" href="#t1160">1160</a></span><span class="t"><span class="str">            hf_model: If you have already loaded in the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1161" href="#t1161">1161</a></span><span class="t"><span class="str">                HuggingFace model, you can pass it in here rather than needing to recreate the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1162" href="#t1162">1162</a></span><span class="t"><span class="str">                object. Defaults to None.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1163" href="#t1163">1163</a></span><span class="t"><span class="str">            device: The device to load the model onto. By</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1164" href="#t1164">1164</a></span><span class="t"><span class="str">                default will load to CUDA if available, else CPU.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1165" href="#t1165">1165</a></span><span class="t"><span class="str">            n_devices: The number of devices to split the model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1166" href="#t1166">1166</a></span><span class="t"><span class="str">                across. Defaults to 1. If greater than 1, `device` must be cuda.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1167" href="#t1167">1167</a></span><span class="t"><span class="str">            tokenizer: The tokenizer to use for the model. If not</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1168" href="#t1168">1168</a></span><span class="t"><span class="str">                provided, it is inferred from cfg.tokenizer_name or initialized to None. If None,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1169" href="#t1169">1169</a></span><span class="t"><span class="str">                then the model cannot be passed strings, and d_vocab must be explicitly set.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1170" href="#t1170">1170</a></span><span class="t"><span class="str">            move_to_device: Whether to move the model to the device specified in</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1171" href="#t1171">1171</a></span><span class="t"><span class="str">                cfg. device. Must be true if `n_devices` in the config is greater than 1, since the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1172" href="#t1172">1172</a></span><span class="t"><span class="str">                model's layers will be split across multiple devices.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1173" href="#t1173">1173</a></span><span class="t"><span class="str">            fold_value_biases: Each attention head has a value bias. Values are averaged to create</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1174" href="#t1174">1174</a></span><span class="t"><span class="str">                mixed values (``z``), weighted by the attention pattern, but as the bias is</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1175" href="#t1175">1175</a></span><span class="t"><span class="str">                constant, its contribution to ``z`` is exactly the same. The output of a head is ``z</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1176" href="#t1176">1176</a></span><span class="t"><span class="str">                @ W_O``, and so the value bias just linearly adds to the output of the head. This</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1177" href="#t1177">1177</a></span><span class="t"><span class="str">                means that the value bias of a head has nothing to do with the head, and is just a</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1178" href="#t1178">1178</a></span><span class="t"><span class="str">                constant added to the attention layer outputs. We can take the sum across these and</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1179" href="#t1179">1179</a></span><span class="t"><span class="str">                b_O to get an "effective bias" for the layer. In code, we set ``b_V=0``. and ``b_O =</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1180" href="#t1180">1180</a></span><span class="t"><span class="str">                (b_V @ W_O).sum(dim=0) + b_O``.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1181" href="#t1181">1181</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1182" href="#t1182">1182</a></span><span class="t"><span class="str">                The technical derivation of this is as follows. ``v = residual @ W_V[h] +</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1183" href="#t1183">1183</a></span><span class="t"><span class="str">                broadcast_b_V[h]`` for each head ``h`` (where ``b_V`` is broadcast up from shape</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1184" href="#t1184">1184</a></span><span class="t"><span class="str">                ``d_head`` to shape ``[position, d_head]``). And ``z = pattern[h] @ v = pattern[h] @</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1185" href="#t1185">1185</a></span><span class="t"><span class="str">                residual @ W_V[h] + pattern[h] @ broadcast_b_V[h]``. Because ``pattern[h]`` is</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1186" href="#t1186">1186</a></span><span class="t"><span class="str">                ``[destination_position, source_position]`` and ``broadcast_b_V`` is constant along</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1187" href="#t1187">1187</a></span><span class="t"><span class="str">                the ``(source_)position`` dimension, we're basically just multiplying it by the sum</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1188" href="#t1188">1188</a></span><span class="t"><span class="str">                of the pattern across the ``source_position`` dimension, which is just ``1``. So it</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1189" href="#t1189">1189</a></span><span class="t"><span class="str">                remains exactly the same, and so is just broadcast across the destination positions.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1190" href="#t1190">1190</a></span><span class="t"><span class="str">            default_prepend_bos: Default behavior of whether to prepend the BOS</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1191" href="#t1191">1191</a></span><span class="t"><span class="str">                token when the methods of HookedTransformer process input text to tokenize (only</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1192" href="#t1192">1192</a></span><span class="t"><span class="str">                when input is a string). Defaults to True - even for models not explicitly trained</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1193" href="#t1193">1193</a></span><span class="t"><span class="str">                with this, heads often use the first position as a resting position and accordingly</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1194" href="#t1194">1194</a></span><span class="t"><span class="str">                lose information from the first token, so this empirically seems to give better</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1195" href="#t1195">1195</a></span><span class="t"><span class="str">                results. To change the default behavior to False, pass in default_prepend_bos=False.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1196" href="#t1196">1196</a></span><span class="t"><span class="str">                Note that you can also locally override the default behavior by passing in</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1197" href="#t1197">1197</a></span><span class="t"><span class="str">                prepend_bos=True/False when you call a method that processes the input string.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1198" href="#t1198">1198</a></span><span class="t"><span class="str">            from_pretrained_kwargs: Any other optional argument passed to</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1199" href="#t1199">1199</a></span><span class="t"><span class="str">                HuggingFace's from_pretrained (e.g. "cache_dir" or "torch_dtype"). Also passed to</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1200" href="#t1200">1200</a></span><span class="t"><span class="str">                other HuggingFace functions when compatible. For some models or arguments it doesn't</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1201" href="#t1201">1201</a></span><span class="t"><span class="str">                work, especially for models that are not internally loaded with HuggingFace's</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1202" href="#t1202">1202</a></span><span class="t"><span class="str">                from_pretrained (e.g. SoLU models).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1203" href="#t1203">1203</a></span><span class="t"><span class="str">            dtype: What data type to load the model in (also sets the dtype of</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1204" href="#t1204">1204</a></span><span class="t"><span class="str">                the HuggingFace model). Set to bfloat16 or float16 if you get out of memory errors when loading</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1205" href="#t1205">1205</a></span><span class="t"><span class="str">                the model.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1206" href="#t1206">1206</a></span><span class="t"><span class="str">            default_padding_side: Which side to pad on when tokenizing. Defaults to</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1207" href="#t1207">1207</a></span><span class="t"><span class="str">                "right".</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1208" href="#t1208">1208</a></span><span class="t"><span class="str">            first_n_layers: If specified, only load the first n layers of the model.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1209" href="#t1209">1209</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1210" href="#t1210">1210</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1211" href="#t1211">1211</a></span><span class="t">        <span class="key">assert</span> <span class="key">not</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1212" href="#t1212">1212</a></span><span class="t">            <span class="nam">from_pretrained_kwargs</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"load_in_8bit"</span><span class="op">,</span> <span class="key">False</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1213" href="#t1213">1213</a></span><span class="t">            <span class="key">or</span> <span class="nam">from_pretrained_kwargs</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"load_in_4bit"</span><span class="op">,</span> <span class="key">False</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1214" href="#t1214">1214</a></span><span class="t">        <span class="op">)</span><span class="op">,</span> <span class="str">"Quantization not supported"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1215" href="#t1215">1215</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1216" href="#t1216">1216</a></span><span class="t">        <span class="key">if</span> <span class="nam">hf_model</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1216&#x202F;&#x219B;&#x202F;1217</span><span class="annotate long">line 1216 didn't jump to line 1217, because the condition on line 1216 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1217" href="#t1217">1217</a></span><span class="t">            <span class="nam">hf_cfg</span> <span class="op">=</span> <span class="nam">hf_model</span><span class="op">.</span><span class="nam">config</span><span class="op">.</span><span class="nam">to_dict</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1218" href="#t1218">1218</a></span><span class="t">            <span class="nam">qc</span> <span class="op">=</span> <span class="nam">hf_cfg</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"quantization_config"</span><span class="op">,</span> <span class="op">{</span><span class="op">}</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1219" href="#t1219">1219</a></span><span class="t">            <span class="nam">load_in_4bit</span> <span class="op">=</span> <span class="nam">qc</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"load_in_4bit"</span><span class="op">,</span> <span class="key">False</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1220" href="#t1220">1220</a></span><span class="t">            <span class="nam">load_in_8bit</span> <span class="op">=</span> <span class="nam">qc</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"load_in_8bit"</span><span class="op">,</span> <span class="key">False</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1221" href="#t1221">1221</a></span><span class="t">            <span class="nam">quant_method</span> <span class="op">=</span> <span class="nam">qc</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"quant_method"</span><span class="op">,</span> <span class="str">""</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1222" href="#t1222">1222</a></span><span class="t">            <span class="key">assert</span> <span class="key">not</span> <span class="nam">load_in_8bit</span><span class="op">,</span> <span class="str">"8-bit quantization is not supported"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1223" href="#t1223">1223</a></span><span class="t">            <span class="key">assert</span> <span class="key">not</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1224" href="#t1224">1224</a></span><span class="t">                <span class="nam">load_in_4bit</span> <span class="key">and</span> <span class="op">(</span><span class="nam">version</span><span class="op">.</span><span class="nam">parse</span><span class="op">(</span><span class="nam">torch</span><span class="op">.</span><span class="nam">__version__</span><span class="op">)</span> <span class="op">&lt;</span> <span class="nam">version</span><span class="op">.</span><span class="nam">parse</span><span class="op">(</span><span class="str">"2.1.1"</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1225" href="#t1225">1225</a></span><span class="t">            <span class="op">)</span><span class="op">,</span> <span class="str">"Quantization is only supported for torch versions >= 2.1.1"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1226" href="#t1226">1226</a></span><span class="t">            <span class="key">assert</span> <span class="key">not</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1227" href="#t1227">1227</a></span><span class="t">                <span class="nam">load_in_4bit</span> <span class="key">and</span> <span class="op">(</span><span class="str">"llama"</span> <span class="key">not</span> <span class="key">in</span> <span class="nam">model_name</span><span class="op">.</span><span class="nam">lower</span><span class="op">(</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1228" href="#t1228">1228</a></span><span class="t">            <span class="op">)</span><span class="op">,</span> <span class="str">"Quantization is only supported for Llama models"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1229" href="#t1229">1229</a></span><span class="t">            <span class="key">if</span> <span class="nam">load_in_4bit</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1230" href="#t1230">1230</a></span><span class="t">                <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1231" href="#t1231">1231</a></span><span class="t">                    <span class="nam">qc</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"quant_method"</span><span class="op">,</span> <span class="str">""</span><span class="op">)</span> <span class="op">==</span> <span class="str">"bitsandbytes"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1232" href="#t1232">1232</a></span><span class="t">                <span class="op">)</span><span class="op">,</span> <span class="str">"Only bitsandbytes quantization is supported"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1233" href="#t1233">1233</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1234" href="#t1234">1234</a></span><span class="t">            <span class="nam">hf_cfg</span> <span class="op">=</span> <span class="op">{</span><span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1235" href="#t1235">1235</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1236" href="#t1236">1236</a></span><span class="t">        <span class="key">if</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">dtype</span><span class="op">,</span> <span class="nam">str</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1237" href="#t1237">1237</a></span><span class="t">            <span class="com"># Convert from string to a torch dtype</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1238" href="#t1238">1238</a></span><span class="t">            <span class="nam">dtype</span> <span class="op">=</span> <span class="nam">DTYPE_FROM_STRING</span><span class="op">[</span><span class="nam">dtype</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1239" href="#t1239">1239</a></span><span class="t">        <span class="key">if</span> <span class="str">"torch_dtype"</span> <span class="key">in</span> <span class="nam">from_pretrained_kwargs</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1239&#x202F;&#x219B;&#x202F;1242</span><span class="annotate long">line 1239 didn't jump to line 1242, because the condition on line 1239 was never true</span></span></p>
+    <p class="pln"><span class="n"><a id="t1240" href="#t1240">1240</a></span><span class="t">            <span class="com"># For backwards compatibility with the previous way to do low precision loading</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1241" href="#t1241">1241</a></span><span class="t">            <span class="com"># This should maybe check the user did not explicitly set dtype *and* torch_dtype</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1242" href="#t1242">1242</a></span><span class="t">            <span class="nam">dtype</span> <span class="op">=</span> <span class="nam">from_pretrained_kwargs</span><span class="op">[</span><span class="str">"torch_dtype"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1243" href="#t1243">1243</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1244" href="#t1244">1244</a></span><span class="t">        <span class="key">if</span> <span class="op">(</span>&nbsp;</span><span class="r"><span class="annotate short">1244&#x202F;&#x219B;&#x202F;1248</span><span class="annotate long">line 1244 didn't jump to line 1248, because the condition on line 1244 was never true</span></span></p>
+    <p class="pln"><span class="n"><a id="t1245" href="#t1245">1245</a></span><span class="t">            <span class="op">(</span><span class="nam">from_pretrained_kwargs</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"torch_dtype"</span><span class="op">,</span> <span class="key">None</span><span class="op">)</span> <span class="op">==</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">float16</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1246" href="#t1246">1246</a></span><span class="t">            <span class="key">or</span> <span class="nam">dtype</span> <span class="op">==</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">float16</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1247" href="#t1247">1247</a></span><span class="t">        <span class="op">)</span> <span class="key">and</span> <span class="nam">device</span> <span class="key">in</span> <span class="op">[</span><span class="str">"cpu"</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1248" href="#t1248">1248</a></span><span class="t">            <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span><span class="str">"float16 models may not work on CPU. Consider using a GPU or bfloat16."</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1249" href="#t1249">1249</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1250" href="#t1250">1250</a></span><span class="t">        <span class="com"># Get the model name used in HuggingFace, rather than the alias.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1251" href="#t1251">1251</a></span><span class="t">        <span class="nam">official_model_name</span> <span class="op">=</span> <span class="nam">loading</span><span class="op">.</span><span class="nam">get_official_model_name</span><span class="op">(</span><span class="nam">model_name</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1252" href="#t1252">1252</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1253" href="#t1253">1253</a></span><span class="t">        <span class="com"># Load the config into an HookedTransformerConfig object. If loading from a</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1254" href="#t1254">1254</a></span><span class="t">        <span class="com"># checkpoint, the config object will contain the information about the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1255" href="#t1255">1255</a></span><span class="t">        <span class="com"># checkpoint</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1256" href="#t1256">1256</a></span><span class="t">        <span class="nam">cfg</span> <span class="op">=</span> <span class="nam">loading</span><span class="op">.</span><span class="nam">get_pretrained_model_config</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1257" href="#t1257">1257</a></span><span class="t">            <span class="nam">official_model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1258" href="#t1258">1258</a></span><span class="t">            <span class="nam">hf_cfg</span><span class="op">=</span><span class="nam">hf_cfg</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1259" href="#t1259">1259</a></span><span class="t">            <span class="nam">checkpoint_index</span><span class="op">=</span><span class="nam">checkpoint_index</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1260" href="#t1260">1260</a></span><span class="t">            <span class="nam">checkpoint_value</span><span class="op">=</span><span class="nam">checkpoint_value</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1261" href="#t1261">1261</a></span><span class="t">            <span class="nam">fold_ln</span><span class="op">=</span><span class="nam">fold_ln</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1262" href="#t1262">1262</a></span><span class="t">            <span class="nam">device</span><span class="op">=</span><span class="nam">device</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1263" href="#t1263">1263</a></span><span class="t">            <span class="nam">n_devices</span><span class="op">=</span><span class="nam">n_devices</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1264" href="#t1264">1264</a></span><span class="t">            <span class="nam">default_prepend_bos</span><span class="op">=</span><span class="nam">default_prepend_bos</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1265" href="#t1265">1265</a></span><span class="t">            <span class="nam">dtype</span><span class="op">=</span><span class="nam">dtype</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1266" href="#t1266">1266</a></span><span class="t">            <span class="nam">first_n_layers</span><span class="op">=</span><span class="nam">first_n_layers</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1267" href="#t1267">1267</a></span><span class="t">            <span class="op">**</span><span class="nam">from_pretrained_kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1268" href="#t1268">1268</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1269" href="#t1269">1269</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1270" href="#t1270">1270</a></span><span class="t">        <span class="key">if</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">positional_embedding_type</span> <span class="op">==</span> <span class="str">"shortformer"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1271" href="#t1271">1271</a></span><span class="t">            <span class="key">if</span> <span class="nam">fold_ln</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1272" href="#t1272">1272</a></span><span class="t">                <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1273" href="#t1273">1273</a></span><span class="t">                    <span class="str">"You tried to specify fold_ln=True for a shortformer model, but this can't be done! Setting fold_"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1274" href="#t1274">1274</a></span><span class="t">                    <span class="str">"ln=False instead."</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1275" href="#t1275">1275</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1276" href="#t1276">1276</a></span><span class="t">                <span class="nam">fold_ln</span> <span class="op">=</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1277" href="#t1277">1277</a></span><span class="t">            <span class="key">if</span> <span class="nam">center_unembed</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1278" href="#t1278">1278</a></span><span class="t">                <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1279" href="#t1279">1279</a></span><span class="t">                    <span class="str">"You tried to specify center_unembed=True for a shortformer model, but this can't be done! "</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1280" href="#t1280">1280</a></span><span class="t">                    <span class="str">"Setting center_unembed=False instead."</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1281" href="#t1281">1281</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1282" href="#t1282">1282</a></span><span class="t">                <span class="nam">center_unembed</span> <span class="op">=</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1283" href="#t1283">1283</a></span><span class="t">            <span class="key">if</span> <span class="nam">center_writing_weights</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1284" href="#t1284">1284</a></span><span class="t">                <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1285" href="#t1285">1285</a></span><span class="t">                    <span class="str">"You tried to specify center_writing_weights=True for a shortformer model, but this can't be done! "</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1286" href="#t1286">1286</a></span><span class="t">                    <span class="str">"Setting center_writing_weights=False instead."</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1287" href="#t1287">1287</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1288" href="#t1288">1288</a></span><span class="t">                <span class="nam">center_writing_weights</span> <span class="op">=</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1289" href="#t1289">1289</a></span><span class="t">        <span class="key">if</span> <span class="nam">center_unembed</span> <span class="key">and</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">output_logits_soft_cap</span> <span class="op">></span> <span class="num">0.0</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1289&#x202F;&#x219B;&#x202F;1290</span><span class="annotate long">line 1289 didn't jump to line 1290, because the condition on line 1289 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1290" href="#t1290">1290</a></span><span class="t">            <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1291" href="#t1291">1291</a></span><span class="t">                <span class="str">"You tried to specify center_unembed=True for a model using logit softcap, but this can't be done! Softcapping is not invariant upon adding a constant"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1292" href="#t1292">1292</a></span><span class="t">                <span class="str">"Setting center_unembed=False instead."</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1293" href="#t1293">1293</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1294" href="#t1294">1294</a></span><span class="t">            <span class="nam">center_unembed</span> <span class="op">=</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1295" href="#t1295">1295</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1296" href="#t1296">1296</a></span><span class="t">        <span class="com"># Get the state dict of the model (ie a mapping of parameter names to tensors), processed to</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1297" href="#t1297">1297</a></span><span class="t">        <span class="com"># match the HookedTransformer parameter names.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1298" href="#t1298">1298</a></span><span class="t">        <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">loading</span><span class="op">.</span><span class="nam">get_pretrained_state_dict</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1299" href="#t1299">1299</a></span><span class="t">            <span class="nam">official_model_name</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">,</span> <span class="nam">hf_model</span><span class="op">,</span> <span class="nam">dtype</span><span class="op">=</span><span class="nam">dtype</span><span class="op">,</span> <span class="op">**</span><span class="nam">from_pretrained_kwargs</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1300" href="#t1300">1300</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1301" href="#t1301">1301</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1302" href="#t1302">1302</a></span><span class="t">        <span class="com"># Create the HookedTransformer object</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1303" href="#t1303">1303</a></span><span class="t">        <span class="nam">model</span> <span class="op">=</span> <span class="nam">cls</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1304" href="#t1304">1304</a></span><span class="t">            <span class="nam">cfg</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1305" href="#t1305">1305</a></span><span class="t">            <span class="nam">tokenizer</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1306" href="#t1306">1306</a></span><span class="t">            <span class="nam">move_to_device</span><span class="op">=</span><span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1307" href="#t1307">1307</a></span><span class="t">            <span class="nam">default_padding_side</span><span class="op">=</span><span class="nam">default_padding_side</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1308" href="#t1308">1308</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1309" href="#t1309">1309</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1310" href="#t1310">1310</a></span><span class="t">        <span class="nam">model</span><span class="op">.</span><span class="nam">load_and_process_state_dict</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1311" href="#t1311">1311</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1312" href="#t1312">1312</a></span><span class="t">            <span class="nam">fold_ln</span><span class="op">=</span><span class="nam">fold_ln</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1313" href="#t1313">1313</a></span><span class="t">            <span class="nam">center_writing_weights</span><span class="op">=</span><span class="nam">center_writing_weights</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1314" href="#t1314">1314</a></span><span class="t">            <span class="nam">center_unembed</span><span class="op">=</span><span class="nam">center_unembed</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1315" href="#t1315">1315</a></span><span class="t">            <span class="nam">fold_value_biases</span><span class="op">=</span><span class="nam">fold_value_biases</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1316" href="#t1316">1316</a></span><span class="t">            <span class="nam">refactor_factored_attn_matrices</span><span class="op">=</span><span class="nam">refactor_factored_attn_matrices</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1317" href="#t1317">1317</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1318" href="#t1318">1318</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1319" href="#t1319">1319</a></span><span class="t">        <span class="nam">print</span><span class="op">(</span><span class="str">f"Loaded pretrained model {model_name} into HookedTransformer"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1320" href="#t1320">1320</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1321" href="#t1321">1321</a></span><span class="t">        <span class="key">return</span> <span class="nam">model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1322" href="#t1322">1322</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1323" href="#t1323">1323</a></span><span class="t">    <span class="op">@</span><span class="nam">classmethod</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1324" href="#t1324">1324</a></span><span class="t">    <span class="key">def</span> <span class="nam">from_pretrained_no_processing</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1325" href="#t1325">1325</a></span><span class="t">        <span class="nam">cls</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1326" href="#t1326">1326</a></span><span class="t">        <span class="nam">model_name</span><span class="op">:</span> <span class="nam">str</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1327" href="#t1327">1327</a></span><span class="t">        <span class="nam">fold_ln</span><span class="op">=</span><span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1328" href="#t1328">1328</a></span><span class="t">        <span class="nam">center_writing_weights</span><span class="op">=</span><span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1329" href="#t1329">1329</a></span><span class="t">        <span class="nam">center_unembed</span><span class="op">=</span><span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1330" href="#t1330">1330</a></span><span class="t">        <span class="nam">refactor_factored_attn_matrices</span><span class="op">=</span><span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1331" href="#t1331">1331</a></span><span class="t">        <span class="nam">fold_value_biases</span><span class="op">=</span><span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1332" href="#t1332">1332</a></span><span class="t">        <span class="nam">dtype</span><span class="op">=</span><span class="nam">torch</span><span class="op">.</span><span class="nam">float32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1333" href="#t1333">1333</a></span><span class="t">        <span class="nam">default_prepend_bos</span><span class="op">=</span><span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1334" href="#t1334">1334</a></span><span class="t">        <span class="nam">default_padding_side</span><span class="op">=</span><span class="str">"right"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1335" href="#t1335">1335</a></span><span class="t">        <span class="op">**</span><span class="nam">from_pretrained_kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1336" href="#t1336">1336</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1337" href="#t1337">1337</a></span><span class="t">        <span class="str">"""Wrapper for from_pretrained.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1338" href="#t1338">1338</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1339" href="#t1339">1339</a></span><span class="t"><span class="str">        Wrapper for from_pretrained with all boolean flags related to simplifying the model set to</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1340" href="#t1340">1340</a></span><span class="t"><span class="str">        False. Refer to from_pretrained for details.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1341" href="#t1341">1341</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1342" href="#t1342">1342</a></span><span class="t">        <span class="key">return</span> <span class="nam">cls</span><span class="op">.</span><span class="nam">from_pretrained</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1343" href="#t1343">1343</a></span><span class="t">            <span class="nam">model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1344" href="#t1344">1344</a></span><span class="t">            <span class="nam">fold_ln</span><span class="op">=</span><span class="nam">fold_ln</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1345" href="#t1345">1345</a></span><span class="t">            <span class="nam">center_writing_weights</span><span class="op">=</span><span class="nam">center_writing_weights</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1346" href="#t1346">1346</a></span><span class="t">            <span class="nam">center_unembed</span><span class="op">=</span><span class="nam">center_unembed</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1347" href="#t1347">1347</a></span><span class="t">            <span class="nam">fold_value_biases</span><span class="op">=</span><span class="nam">fold_value_biases</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1348" href="#t1348">1348</a></span><span class="t">            <span class="nam">refactor_factored_attn_matrices</span><span class="op">=</span><span class="nam">refactor_factored_attn_matrices</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1349" href="#t1349">1349</a></span><span class="t">            <span class="nam">dtype</span><span class="op">=</span><span class="nam">dtype</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1350" href="#t1350">1350</a></span><span class="t">            <span class="nam">default_prepend_bos</span><span class="op">=</span><span class="nam">default_prepend_bos</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1351" href="#t1351">1351</a></span><span class="t">            <span class="nam">default_padding_side</span><span class="op">=</span><span class="nam">default_padding_side</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1352" href="#t1352">1352</a></span><span class="t">            <span class="op">**</span><span class="nam">from_pretrained_kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1353" href="#t1353">1353</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1354" href="#t1354">1354</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1355" href="#t1355">1355</a></span><span class="t">    <span class="key">def</span> <span class="nam">init_weights</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1356" href="#t1356">1356</a></span><span class="t">        <span class="str">"""Initialize weights.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1319" href="#t1319">1319</a></span><span class="t">        <span class="key">if</span> <span class="nam">move_to_device</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1319&#x202F;&#x219B;&#x202F;1322</span><span class="annotate long">line 1319 didn't jump to line 1322, because the condition on line 1319 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t1320" href="#t1320">1320</a></span><span class="t">            <span class="nam">model</span><span class="op">.</span><span class="nam">move_model_modules_to_device</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1321" href="#t1321">1321</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1322" href="#t1322">1322</a></span><span class="t">        <span class="nam">print</span><span class="op">(</span><span class="str">f"Loaded pretrained model {model_name} into HookedTransformer"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1323" href="#t1323">1323</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1324" href="#t1324">1324</a></span><span class="t">        <span class="key">return</span> <span class="nam">model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1325" href="#t1325">1325</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1326" href="#t1326">1326</a></span><span class="t">    <span class="op">@</span><span class="nam">classmethod</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1327" href="#t1327">1327</a></span><span class="t">    <span class="key">def</span> <span class="nam">from_pretrained_no_processing</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1328" href="#t1328">1328</a></span><span class="t">        <span class="nam">cls</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1329" href="#t1329">1329</a></span><span class="t">        <span class="nam">model_name</span><span class="op">:</span> <span class="nam">str</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1330" href="#t1330">1330</a></span><span class="t">        <span class="nam">fold_ln</span><span class="op">=</span><span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1331" href="#t1331">1331</a></span><span class="t">        <span class="nam">center_writing_weights</span><span class="op">=</span><span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1332" href="#t1332">1332</a></span><span class="t">        <span class="nam">center_unembed</span><span class="op">=</span><span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1333" href="#t1333">1333</a></span><span class="t">        <span class="nam">refactor_factored_attn_matrices</span><span class="op">=</span><span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1334" href="#t1334">1334</a></span><span class="t">        <span class="nam">fold_value_biases</span><span class="op">=</span><span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1335" href="#t1335">1335</a></span><span class="t">        <span class="nam">dtype</span><span class="op">=</span><span class="nam">torch</span><span class="op">.</span><span class="nam">float32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1336" href="#t1336">1336</a></span><span class="t">        <span class="nam">default_prepend_bos</span><span class="op">=</span><span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1337" href="#t1337">1337</a></span><span class="t">        <span class="nam">default_padding_side</span><span class="op">=</span><span class="str">"right"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1338" href="#t1338">1338</a></span><span class="t">        <span class="op">**</span><span class="nam">from_pretrained_kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1339" href="#t1339">1339</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1340" href="#t1340">1340</a></span><span class="t">        <span class="str">"""Wrapper for from_pretrained.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1341" href="#t1341">1341</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1342" href="#t1342">1342</a></span><span class="t"><span class="str">        Wrapper for from_pretrained with all boolean flags related to simplifying the model set to</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1343" href="#t1343">1343</a></span><span class="t"><span class="str">        False. Refer to from_pretrained for details.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1344" href="#t1344">1344</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1345" href="#t1345">1345</a></span><span class="t">        <span class="key">return</span> <span class="nam">cls</span><span class="op">.</span><span class="nam">from_pretrained</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1346" href="#t1346">1346</a></span><span class="t">            <span class="nam">model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1347" href="#t1347">1347</a></span><span class="t">            <span class="nam">fold_ln</span><span class="op">=</span><span class="nam">fold_ln</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1348" href="#t1348">1348</a></span><span class="t">            <span class="nam">center_writing_weights</span><span class="op">=</span><span class="nam">center_writing_weights</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1349" href="#t1349">1349</a></span><span class="t">            <span class="nam">center_unembed</span><span class="op">=</span><span class="nam">center_unembed</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1350" href="#t1350">1350</a></span><span class="t">            <span class="nam">fold_value_biases</span><span class="op">=</span><span class="nam">fold_value_biases</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1351" href="#t1351">1351</a></span><span class="t">            <span class="nam">refactor_factored_attn_matrices</span><span class="op">=</span><span class="nam">refactor_factored_attn_matrices</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1352" href="#t1352">1352</a></span><span class="t">            <span class="nam">dtype</span><span class="op">=</span><span class="nam">dtype</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1353" href="#t1353">1353</a></span><span class="t">            <span class="nam">default_prepend_bos</span><span class="op">=</span><span class="nam">default_prepend_bos</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1354" href="#t1354">1354</a></span><span class="t">            <span class="nam">default_padding_side</span><span class="op">=</span><span class="nam">default_padding_side</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1355" href="#t1355">1355</a></span><span class="t">            <span class="op">**</span><span class="nam">from_pretrained_kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1356" href="#t1356">1356</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1357" href="#t1357">1357</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1358" href="#t1358">1358</a></span><span class="t"><span class="str">        LayerNorm weights are already initialized to 1.0, and all biases are initialized to 0.0</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1359" href="#t1359">1359</a></span><span class="t"><span class="str">        (including LayerNorm), so this just initializes weight matrices.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1358" href="#t1358">1358</a></span><span class="t">    <span class="key">def</span> <span class="nam">init_weights</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1359" href="#t1359">1359</a></span><span class="t">        <span class="str">"""Initialize weights.</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1360" href="#t1360">1360</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1361" href="#t1361">1361</a></span><span class="t"><span class="str">        Weight matrices are set to empty by default (to save space + compute, since they're the bulk</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1362" href="#t1362">1362</a></span><span class="t"><span class="str">        of the parameters), so it is important to call this if you are not loading in pretrained</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1363" href="#t1363">1363</a></span><span class="t"><span class="str">        weights! Note that this function assumes that weight names being with `W_`.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1364" href="#t1364">1364</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1365" href="#t1365">1365</a></span><span class="t"><span class="str">        Set seed here to ensure determinism.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1366" href="#t1366">1366</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1367" href="#t1367">1367</a></span><span class="t"><span class="str">        This does NOT follow the PyTorch scheme, which as far as I can tell is super out of date but</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1368" href="#t1368">1368</a></span><span class="t"><span class="str">        no one has gotten round to updating it? https://github.com/pytorch/pytorch/issues/18182</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1361" href="#t1361">1361</a></span><span class="t"><span class="str">        LayerNorm weights are already initialized to 1.0, and all biases are initialized to 0.0</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1362" href="#t1362">1362</a></span><span class="t"><span class="str">        (including LayerNorm), so this just initializes weight matrices.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1363" href="#t1363">1363</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1364" href="#t1364">1364</a></span><span class="t"><span class="str">        Weight matrices are set to empty by default (to save space + compute, since they're the bulk</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1365" href="#t1365">1365</a></span><span class="t"><span class="str">        of the parameters), so it is important to call this if you are not loading in pretrained</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1366" href="#t1366">1366</a></span><span class="t"><span class="str">        weights! Note that this function assumes that weight names being with `W_`.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1367" href="#t1367">1367</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1368" href="#t1368">1368</a></span><span class="t"><span class="str">        Set seed here to ensure determinism.</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1369" href="#t1369">1369</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1370" href="#t1370">1370</a></span><span class="t"><span class="str">        The default PyTorch scheme is the following: all linear layers use uniform(-1/sqrt(fan_in),</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1371" href="#t1371">1371</a></span><span class="t"><span class="str">        1/sqrt(fan_in)) for weights, and uniform(-1/sqrt(fan_in), 1/sqrt(fan_in)) for biases. For</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1372" href="#t1372">1372</a></span><span class="t"><span class="str">        biases, fan_in is computed using the fan_in for the weight matrix of the linear layer. Note</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1373" href="#t1373">1373</a></span><span class="t"><span class="str">        tha it *does not actually* use Kaiming initialization, despite the fact that it calls the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1374" href="#t1374">1374</a></span><span class="t"><span class="str">        function.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1375" href="#t1375">1375</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1376" href="#t1376">1376</a></span><span class="t"><span class="str">        However, for Transformer blocks, it instead initializes biases to zero and weights using Xavier uniform, that</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1377" href="#t1377">1377</a></span><span class="t"><span class="str">        is: uniform(-sqrt(6 / (fan_in + fan_out)), sqrt(6 / (fan_in + fan_out))) for weights.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1370" href="#t1370">1370</a></span><span class="t"><span class="str">        This does NOT follow the PyTorch scheme, which as far as I can tell is super out of date but</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1371" href="#t1371">1371</a></span><span class="t"><span class="str">        no one has gotten round to updating it? https://github.com/pytorch/pytorch/issues/18182</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1372" href="#t1372">1372</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1373" href="#t1373">1373</a></span><span class="t"><span class="str">        The default PyTorch scheme is the following: all linear layers use uniform(-1/sqrt(fan_in),</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1374" href="#t1374">1374</a></span><span class="t"><span class="str">        1/sqrt(fan_in)) for weights, and uniform(-1/sqrt(fan_in), 1/sqrt(fan_in)) for biases. For</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1375" href="#t1375">1375</a></span><span class="t"><span class="str">        biases, fan_in is computed using the fan_in for the weight matrix of the linear layer. Note</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1376" href="#t1376">1376</a></span><span class="t"><span class="str">        tha it *does not actually* use Kaiming initialization, despite the fact that it calls the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1377" href="#t1377">1377</a></span><span class="t"><span class="str">        function.</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1378" href="#t1378">1378</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1379" href="#t1379">1379</a></span><span class="t"><span class="str">        PyTorch Transformers are especially bad - TransformerEncoder initializes all layers to the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1380" href="#t1380">1380</a></span><span class="t"><span class="str">        exact same weights?! https://github.com/pytorch/pytorch/issues/72253.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1379" href="#t1379">1379</a></span><span class="t"><span class="str">        However, for Transformer blocks, it instead initializes biases to zero and weights using Xavier uniform, that</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1380" href="#t1380">1380</a></span><span class="t"><span class="str">        is: uniform(-sqrt(6 / (fan_in + fan_out)), sqrt(6 / (fan_in + fan_out))) for weights.</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1381" href="#t1381">1381</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1382" href="#t1382">1382</a></span><span class="t"><span class="str">        The best paper I've found on transformer initialization is the muP paper, but haven't</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1383" href="#t1383">1383</a></span><span class="t"><span class="str">        integrated those ideas yet: https://arxiv.org/abs/2203.03466</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1382" href="#t1382">1382</a></span><span class="t"><span class="str">        PyTorch Transformers are especially bad - TransformerEncoder initializes all layers to the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1383" href="#t1383">1383</a></span><span class="t"><span class="str">        exact same weights?! https://github.com/pytorch/pytorch/issues/72253.</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1384" href="#t1384">1384</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1385" href="#t1385">1385</a></span><span class="t"><span class="str">        We split off the initialization into separate functions because muP initialization handles</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1386" href="#t1386">1386</a></span><span class="t"><span class="str">        different parts of the model differently.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1387" href="#t1387">1387</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1388" href="#t1388">1388</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1389" href="#t1389">1389</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">seed</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1389&#x202F;&#x219B;&#x202F;1390</span><span class="annotate long">line 1389 didn't jump to line 1390, because the condition on line 1389 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1390" href="#t1390">1390</a></span><span class="t">            <span class="nam">torch</span><span class="op">.</span><span class="nam">manual_seed</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">seed</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1385" href="#t1385">1385</a></span><span class="t"><span class="str">        The best paper I've found on transformer initialization is the muP paper, but haven't</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1386" href="#t1386">1386</a></span><span class="t"><span class="str">        integrated those ideas yet: https://arxiv.org/abs/2203.03466</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1387" href="#t1387">1387</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1388" href="#t1388">1388</a></span><span class="t"><span class="str">        We split off the initialization into separate functions because muP initialization handles</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1389" href="#t1389">1389</a></span><span class="t"><span class="str">        different parts of the model differently.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1390" href="#t1390">1390</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1391" href="#t1391">1391</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1392" href="#t1392">1392</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">init_mode</span> <span class="op">==</span> <span class="str">"gpt2"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1392&#x202F;&#x219B;&#x202F;1394</span><span class="annotate long">line 1392 didn't jump to line 1394, because the condition on line 1392 was never false</span></span></p>
-    <p class="run"><span class="n"><a id="t1393" href="#t1393">1393</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">_init_weights_gpt2</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1394" href="#t1394">1394</a></span><span class="t">        <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">init_mode</span> <span class="op">==</span> <span class="str">"xavier_uniform"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1395" href="#t1395">1395</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">_init_weights_xavier</span><span class="op">(</span><span class="nam">dist_type</span><span class="op">=</span><span class="str">"uniform"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1396" href="#t1396">1396</a></span><span class="t">        <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">init_mode</span> <span class="op">==</span> <span class="str">"xavier_normal"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1397" href="#t1397">1397</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">_init_weights_xavier</span><span class="op">(</span><span class="nam">dist_type</span><span class="op">=</span><span class="str">"normal"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1398" href="#t1398">1398</a></span><span class="t">        <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">init_mode</span> <span class="op">==</span> <span class="str">"kaiming_uniform"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1399" href="#t1399">1399</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">_init_weights_kaiming</span><span class="op">(</span><span class="nam">dist_type</span><span class="op">=</span><span class="str">"uniform"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1400" href="#t1400">1400</a></span><span class="t">        <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">init_mode</span> <span class="op">==</span> <span class="str">"kaiming_normal"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1401" href="#t1401">1401</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">_init_weights_kaiming</span><span class="op">(</span><span class="nam">dist_type</span><span class="op">=</span><span class="str">"normal"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1402" href="#t1402">1402</a></span><span class="t">        <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">init_mode</span> <span class="op">==</span> <span class="str">"muP"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1403" href="#t1403">1403</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">_init_weights_muP</span><span class="op">(</span><span class="nam">dist_type</span><span class="op">=</span><span class="str">"normal"</span><span class="op">)</span>  <span class="com"># muP uses normal initialization</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1404" href="#t1404">1404</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1405" href="#t1405">1405</a></span><span class="t">    <span class="key">def</span> <span class="nam">_init_weights_gpt2</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1406" href="#t1406">1406</a></span><span class="t">        <span class="str">"""Initialize weights with GPT-2 initialization. Biases are initialized to 0.0 and weights</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1407" href="#t1407">1407</a></span><span class="t"><span class="str">        are initialized to N(0, 0.64/d_model) if initializer_range is not set, otherwise std is initializer_range.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1408" href="#t1408">1408</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1409" href="#t1409">1409</a></span><span class="t">        <span class="key">for</span> <span class="nam">name</span><span class="op">,</span> <span class="nam">param</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">named_parameters</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1410" href="#t1410">1410</a></span><span class="t">            <span class="key">if</span> <span class="str">"W_"</span> <span class="key">in</span> <span class="nam">name</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1411" href="#t1411">1411</a></span><span class="t">                <span class="nam">nn</span><span class="op">.</span><span class="nam">init</span><span class="op">.</span><span class="nam">normal_</span><span class="op">(</span><span class="nam">param</span><span class="op">,</span> <span class="nam">std</span><span class="op">=</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">initializer_range</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1412" href="#t1412">1412</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1413" href="#t1413">1413</a></span><span class="t">    <span class="key">def</span> <span class="nam">_init_weights_xavier</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">dist_type</span><span class="op">=</span><span class="str">"normal"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1414" href="#t1414">1414</a></span><span class="t">        <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1415" href="#t1415">1415</a></span><span class="t"><span class="str">        Initialize weights with Xavier initialization -- that is, scale the weights by sqrt(6 /</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1416" href="#t1416">1416</a></span><span class="t"><span class="str">        (fan_in + fan_out)) for a [-1, 1] uniform distribution, or sqrt(2 / (fan_in + fan_out)) for a</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1417" href="#t1417">1417</a></span><span class="t"><span class="str">        standard normal.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1418" href="#t1418">1418</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1419" href="#t1419">1419</a></span><span class="t"><span class="str">        Note that since TransformerLens implements the matrices in the opposite orientation to what</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1420" href="#t1420">1420</a></span><span class="t"><span class="str">        torch does (e.g. it's d_in x d_out, not d_out x d_in as in torch), we need to calculate it</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1421" href="#t1421">1421</a></span><span class="t"><span class="str">        ourselves.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1422" href="#t1422">1422</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1423" href="#t1423">1423</a></span><span class="t">        <span class="nam">gain</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">initializer_range</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1424" href="#t1424">1424</a></span><span class="t">        <span class="key">for</span> <span class="nam">name</span><span class="op">,</span> <span class="nam">param</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">named_parameters</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1425" href="#t1425">1425</a></span><span class="t">            <span class="key">if</span> <span class="str">"W_"</span> <span class="key">in</span> <span class="nam">name</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1426" href="#t1426">1426</a></span><span class="t">                <span class="key">if</span> <span class="nam">dist_type</span> <span class="op">==</span> <span class="str">"uniform"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1427" href="#t1427">1427</a></span><span class="t">                    <span class="nam">init_xavier_uniform_</span><span class="op">(</span><span class="nam">param</span><span class="op">,</span> <span class="nam">gain</span><span class="op">=</span><span class="nam">gain</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1428" href="#t1428">1428</a></span><span class="t">                <span class="key">elif</span> <span class="nam">dist_type</span> <span class="op">==</span> <span class="str">"normal"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1429" href="#t1429">1429</a></span><span class="t">                    <span class="nam">init_xavier_normal_</span><span class="op">(</span><span class="nam">param</span><span class="op">,</span> <span class="nam">gain</span><span class="op">=</span><span class="nam">gain</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1430" href="#t1430">1430</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1431" href="#t1431">1431</a></span><span class="t">    <span class="key">def</span> <span class="nam">_init_weights_kaiming</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">dist_type</span><span class="op">=</span><span class="str">"uniform"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1432" href="#t1432">1432</a></span><span class="t">        <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1433" href="#t1433">1433</a></span><span class="t"><span class="str">        Initialize weights with Kaiming initialization -- that is, scale the weights by</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1434" href="#t1434">1434</a></span><span class="t"><span class="str">        c / sqrt(fan_in), where c = sqrt(2) if the params were immediately preceded by a relu and 1 for</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1435" href="#t1435">1435</a></span><span class="t"><span class="str">        everything else.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1436" href="#t1436">1436</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1437" href="#t1437">1437</a></span><span class="t"><span class="str">        Note that the numbers are actually incorrect here when you're using a nonlinearity other</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1438" href="#t1438">1438</a></span><span class="t"><span class="str">        than relu, e.g. the correct c for SiLu is ~1.74, for tanh it's 5/3 ~= 1.67, and for GeLU it's ~1.57.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1439" href="#t1439">1439</a></span><span class="t"><span class="str">        But this is unlikely to matter in practice.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1440" href="#t1440">1440</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1441" href="#t1441">1441</a></span><span class="t"><span class="str">        I'm just using fan_mode = "fan_in" for now, but it should be trivial to add fan_out.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1442" href="#t1442">1442</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1443" href="#t1443">1443</a></span><span class="t"><span class="str">        Again, we have to implement it ourselves because of the orientation of the matrices.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1444" href="#t1444">1444</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1445" href="#t1445">1445</a></span><span class="t">        <span class="nam">gain</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">initializer_range</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1446" href="#t1446">1446</a></span><span class="t">        <span class="key">for</span> <span class="nam">name</span><span class="op">,</span> <span class="nam">param</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">named_parameters</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1447" href="#t1447">1447</a></span><span class="t">            <span class="key">if</span> <span class="str">"W_"</span> <span class="key">in</span> <span class="nam">name</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1448" href="#t1448">1448</a></span><span class="t">                <span class="key">if</span> <span class="nam">dist_type</span> <span class="op">==</span> <span class="str">"uniform"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1449" href="#t1449">1449</a></span><span class="t">                    <span class="nam">init_kaiming_uniform_</span><span class="op">(</span><span class="nam">param</span><span class="op">,</span> <span class="nam">gain</span><span class="op">=</span><span class="nam">gain</span><span class="op">,</span> <span class="nam">nonlinearity</span><span class="op">=</span><span class="str">"relu"</span><span class="op">,</span> <span class="nam">mode</span><span class="op">=</span><span class="str">"fan_in"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1450" href="#t1450">1450</a></span><span class="t">                <span class="key">elif</span> <span class="nam">dist_type</span> <span class="op">==</span> <span class="str">"normal"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1451" href="#t1451">1451</a></span><span class="t">                    <span class="nam">init_kaiming_normal_</span><span class="op">(</span><span class="nam">param</span><span class="op">,</span> <span class="nam">gain</span><span class="op">=</span><span class="nam">gain</span><span class="op">,</span> <span class="nam">nonlinearity</span><span class="op">=</span><span class="str">"relu"</span><span class="op">,</span> <span class="nam">mode</span><span class="op">=</span><span class="str">"fan_in"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1452" href="#t1452">1452</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1453" href="#t1453">1453</a></span><span class="t">    <span class="key">def</span> <span class="nam">_init_weights_muP</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">dist_type</span><span class="op">=</span><span class="str">"uniform"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1454" href="#t1454">1454</a></span><span class="t">        <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1455" href="#t1455">1455</a></span><span class="t"><span class="str">        Initialize weights with muParameterization. This involves scaling output weights by a factor</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1456" href="#t1456">1456</a></span><span class="t"><span class="str">        of 1/fan_in, input weights and biases by 1, everything else by a factor of 1/sqrt(fan_in).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1457" href="#t1457">1457</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1458" href="#t1458">1458</a></span><span class="t"><span class="str">        Also, you need to use muAdamW, which rescales the learning rate for output weights and</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1459" href="#t1459">1459</a></span><span class="t"><span class="str">        hidden weights by a factor of 1/fan_in.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1392" href="#t1392">1392</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">seed</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1392&#x202F;&#x219B;&#x202F;1393</span><span class="annotate long">line 1392 didn't jump to line 1393, because the condition on line 1392 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1393" href="#t1393">1393</a></span><span class="t">            <span class="nam">torch</span><span class="op">.</span><span class="nam">manual_seed</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">seed</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1394" href="#t1394">1394</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1395" href="#t1395">1395</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">init_mode</span> <span class="op">==</span> <span class="str">"gpt2"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1395&#x202F;&#x219B;&#x202F;1397</span><span class="annotate long">line 1395 didn't jump to line 1397, because the condition on line 1395 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t1396" href="#t1396">1396</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">_init_weights_gpt2</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1397" href="#t1397">1397</a></span><span class="t">        <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">init_mode</span> <span class="op">==</span> <span class="str">"xavier_uniform"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1398" href="#t1398">1398</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">_init_weights_xavier</span><span class="op">(</span><span class="nam">dist_type</span><span class="op">=</span><span class="str">"uniform"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1399" href="#t1399">1399</a></span><span class="t">        <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">init_mode</span> <span class="op">==</span> <span class="str">"xavier_normal"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1400" href="#t1400">1400</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">_init_weights_xavier</span><span class="op">(</span><span class="nam">dist_type</span><span class="op">=</span><span class="str">"normal"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1401" href="#t1401">1401</a></span><span class="t">        <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">init_mode</span> <span class="op">==</span> <span class="str">"kaiming_uniform"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1402" href="#t1402">1402</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">_init_weights_kaiming</span><span class="op">(</span><span class="nam">dist_type</span><span class="op">=</span><span class="str">"uniform"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1403" href="#t1403">1403</a></span><span class="t">        <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">init_mode</span> <span class="op">==</span> <span class="str">"kaiming_normal"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1404" href="#t1404">1404</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">_init_weights_kaiming</span><span class="op">(</span><span class="nam">dist_type</span><span class="op">=</span><span class="str">"normal"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1405" href="#t1405">1405</a></span><span class="t">        <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">init_mode</span> <span class="op">==</span> <span class="str">"muP"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1406" href="#t1406">1406</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">_init_weights_muP</span><span class="op">(</span><span class="nam">dist_type</span><span class="op">=</span><span class="str">"normal"</span><span class="op">)</span>  <span class="com"># muP uses normal initialization</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1407" href="#t1407">1407</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1408" href="#t1408">1408</a></span><span class="t">    <span class="key">def</span> <span class="nam">_init_weights_gpt2</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1409" href="#t1409">1409</a></span><span class="t">        <span class="str">"""Initialize weights with GPT-2 initialization. Biases are initialized to 0.0 and weights</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1410" href="#t1410">1410</a></span><span class="t"><span class="str">        are initialized to N(0, 0.64/d_model) if initializer_range is not set, otherwise std is initializer_range.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1411" href="#t1411">1411</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1412" href="#t1412">1412</a></span><span class="t">        <span class="key">for</span> <span class="nam">name</span><span class="op">,</span> <span class="nam">param</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">named_parameters</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1413" href="#t1413">1413</a></span><span class="t">            <span class="key">if</span> <span class="str">"W_"</span> <span class="key">in</span> <span class="nam">name</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1414" href="#t1414">1414</a></span><span class="t">                <span class="nam">nn</span><span class="op">.</span><span class="nam">init</span><span class="op">.</span><span class="nam">normal_</span><span class="op">(</span><span class="nam">param</span><span class="op">,</span> <span class="nam">std</span><span class="op">=</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">initializer_range</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1415" href="#t1415">1415</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1416" href="#t1416">1416</a></span><span class="t">    <span class="key">def</span> <span class="nam">_init_weights_xavier</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">dist_type</span><span class="op">=</span><span class="str">"normal"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1417" href="#t1417">1417</a></span><span class="t">        <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1418" href="#t1418">1418</a></span><span class="t"><span class="str">        Initialize weights with Xavier initialization -- that is, scale the weights by sqrt(6 /</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1419" href="#t1419">1419</a></span><span class="t"><span class="str">        (fan_in + fan_out)) for a [-1, 1] uniform distribution, or sqrt(2 / (fan_in + fan_out)) for a</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1420" href="#t1420">1420</a></span><span class="t"><span class="str">        standard normal.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1421" href="#t1421">1421</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1422" href="#t1422">1422</a></span><span class="t"><span class="str">        Note that since TransformerLens implements the matrices in the opposite orientation to what</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1423" href="#t1423">1423</a></span><span class="t"><span class="str">        torch does (e.g. it's d_in x d_out, not d_out x d_in as in torch), we need to calculate it</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1424" href="#t1424">1424</a></span><span class="t"><span class="str">        ourselves.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1425" href="#t1425">1425</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1426" href="#t1426">1426</a></span><span class="t">        <span class="nam">gain</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">initializer_range</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1427" href="#t1427">1427</a></span><span class="t">        <span class="key">for</span> <span class="nam">name</span><span class="op">,</span> <span class="nam">param</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">named_parameters</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1428" href="#t1428">1428</a></span><span class="t">            <span class="key">if</span> <span class="str">"W_"</span> <span class="key">in</span> <span class="nam">name</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1429" href="#t1429">1429</a></span><span class="t">                <span class="key">if</span> <span class="nam">dist_type</span> <span class="op">==</span> <span class="str">"uniform"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1430" href="#t1430">1430</a></span><span class="t">                    <span class="nam">init_xavier_uniform_</span><span class="op">(</span><span class="nam">param</span><span class="op">,</span> <span class="nam">gain</span><span class="op">=</span><span class="nam">gain</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1431" href="#t1431">1431</a></span><span class="t">                <span class="key">elif</span> <span class="nam">dist_type</span> <span class="op">==</span> <span class="str">"normal"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1432" href="#t1432">1432</a></span><span class="t">                    <span class="nam">init_xavier_normal_</span><span class="op">(</span><span class="nam">param</span><span class="op">,</span> <span class="nam">gain</span><span class="op">=</span><span class="nam">gain</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1433" href="#t1433">1433</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1434" href="#t1434">1434</a></span><span class="t">    <span class="key">def</span> <span class="nam">_init_weights_kaiming</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">dist_type</span><span class="op">=</span><span class="str">"uniform"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1435" href="#t1435">1435</a></span><span class="t">        <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1436" href="#t1436">1436</a></span><span class="t"><span class="str">        Initialize weights with Kaiming initialization -- that is, scale the weights by</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1437" href="#t1437">1437</a></span><span class="t"><span class="str">        c / sqrt(fan_in), where c = sqrt(2) if the params were immediately preceded by a relu and 1 for</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1438" href="#t1438">1438</a></span><span class="t"><span class="str">        everything else.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1439" href="#t1439">1439</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1440" href="#t1440">1440</a></span><span class="t"><span class="str">        Note that the numbers are actually incorrect here when you're using a nonlinearity other</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1441" href="#t1441">1441</a></span><span class="t"><span class="str">        than relu, e.g. the correct c for SiLu is ~1.74, for tanh it's 5/3 ~= 1.67, and for GeLU it's ~1.57.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1442" href="#t1442">1442</a></span><span class="t"><span class="str">        But this is unlikely to matter in practice.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1443" href="#t1443">1443</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1444" href="#t1444">1444</a></span><span class="t"><span class="str">        I'm just using fan_mode = "fan_in" for now, but it should be trivial to add fan_out.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1445" href="#t1445">1445</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1446" href="#t1446">1446</a></span><span class="t"><span class="str">        Again, we have to implement it ourselves because of the orientation of the matrices.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1447" href="#t1447">1447</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1448" href="#t1448">1448</a></span><span class="t">        <span class="nam">gain</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">initializer_range</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1449" href="#t1449">1449</a></span><span class="t">        <span class="key">for</span> <span class="nam">name</span><span class="op">,</span> <span class="nam">param</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">named_parameters</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1450" href="#t1450">1450</a></span><span class="t">            <span class="key">if</span> <span class="str">"W_"</span> <span class="key">in</span> <span class="nam">name</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1451" href="#t1451">1451</a></span><span class="t">                <span class="key">if</span> <span class="nam">dist_type</span> <span class="op">==</span> <span class="str">"uniform"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1452" href="#t1452">1452</a></span><span class="t">                    <span class="nam">init_kaiming_uniform_</span><span class="op">(</span><span class="nam">param</span><span class="op">,</span> <span class="nam">gain</span><span class="op">=</span><span class="nam">gain</span><span class="op">,</span> <span class="nam">nonlinearity</span><span class="op">=</span><span class="str">"relu"</span><span class="op">,</span> <span class="nam">mode</span><span class="op">=</span><span class="str">"fan_in"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1453" href="#t1453">1453</a></span><span class="t">                <span class="key">elif</span> <span class="nam">dist_type</span> <span class="op">==</span> <span class="str">"normal"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1454" href="#t1454">1454</a></span><span class="t">                    <span class="nam">init_kaiming_normal_</span><span class="op">(</span><span class="nam">param</span><span class="op">,</span> <span class="nam">gain</span><span class="op">=</span><span class="nam">gain</span><span class="op">,</span> <span class="nam">nonlinearity</span><span class="op">=</span><span class="str">"relu"</span><span class="op">,</span> <span class="nam">mode</span><span class="op">=</span><span class="str">"fan_in"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1455" href="#t1455">1455</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1456" href="#t1456">1456</a></span><span class="t">    <span class="key">def</span> <span class="nam">_init_weights_muP</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">dist_type</span><span class="op">=</span><span class="str">"uniform"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1457" href="#t1457">1457</a></span><span class="t">        <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1458" href="#t1458">1458</a></span><span class="t"><span class="str">        Initialize weights with muParameterization. This involves scaling output weights by a factor</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1459" href="#t1459">1459</a></span><span class="t"><span class="str">        of 1/fan_in, input weights and biases by 1, everything else by a factor of 1/sqrt(fan_in).</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1460" href="#t1460">1460</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1461" href="#t1461">1461</a></span><span class="t"><span class="str">        All biases are still assumed to be initialized to 0.0, so we only need to change the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1462" href="#t1462">1462</a></span><span class="t"><span class="str">        weights.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1463" href="#t1463">1463</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1464" href="#t1464">1464</a></span><span class="t">        <span class="key">for</span> <span class="nam">name</span><span class="op">,</span> <span class="nam">param</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">named_parameters</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1465" href="#t1465">1465</a></span><span class="t">            <span class="key">if</span> <span class="str">"W_"</span> <span class="key">in</span> <span class="nam">name</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1466" href="#t1466">1466</a></span><span class="t">                <span class="nam">fan_in</span><span class="op">,</span> <span class="nam">_</span> <span class="op">=</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">calc_fan_in_and_fan_out</span><span class="op">(</span><span class="nam">param</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1467" href="#t1467">1467</a></span><span class="t">                <span class="key">if</span> <span class="str">"embed"</span> <span class="key">in</span> <span class="nam">name</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1468" href="#t1468">1468</a></span><span class="t">                    <span class="nam">scale</span> <span class="op">=</span> <span class="nam">float</span><span class="op">(</span><span class="num">1</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1469" href="#t1469">1469</a></span><span class="t">                <span class="key">elif</span> <span class="str">"unembed"</span> <span class="key">in</span> <span class="nam">name</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1470" href="#t1470">1470</a></span><span class="t">                    <span class="nam">scale</span> <span class="op">=</span> <span class="num">1</span> <span class="op">/</span> <span class="nam">fan_in</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1471" href="#t1471">1471</a></span><span class="t">                <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1472" href="#t1472">1472</a></span><span class="t">                    <span class="nam">scale</span> <span class="op">=</span> <span class="num">1</span> <span class="op">/</span> <span class="nam">fan_in</span><span class="op">**</span><span class="num">0.5</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1473" href="#t1473">1473</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1474" href="#t1474">1474</a></span><span class="t">                <span class="key">if</span> <span class="nam">dist_type</span> <span class="op">==</span> <span class="str">"uniform"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1475" href="#t1475">1475</a></span><span class="t">                    <span class="nam">scale</span> <span class="op">*=</span> <span class="num">3</span><span class="op">**</span><span class="num">0.5</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1476" href="#t1476">1476</a></span><span class="t">                    <span class="nam">nn</span><span class="op">.</span><span class="nam">init</span><span class="op">.</span><span class="nam">uniform_</span><span class="op">(</span><span class="nam">param</span><span class="op">,</span> <span class="op">-</span><span class="nam">scale</span><span class="op">,</span> <span class="nam">scale</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1477" href="#t1477">1477</a></span><span class="t">                <span class="key">elif</span> <span class="nam">dist_type</span> <span class="op">==</span> <span class="str">"normal"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1478" href="#t1478">1478</a></span><span class="t">                    <span class="nam">nn</span><span class="op">.</span><span class="nam">init</span><span class="op">.</span><span class="nam">normal_</span><span class="op">(</span><span class="nam">param</span><span class="op">,</span> <span class="nam">std</span><span class="op">=</span><span class="nam">scale</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1479" href="#t1479">1479</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1480" href="#t1480">1480</a></span><span class="t">    <span class="key">def</span> <span class="nam">load_and_process_state_dict</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1481" href="#t1481">1481</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1482" href="#t1482">1482</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">:</span> <span class="nam">Dict</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1483" href="#t1483">1483</a></span><span class="t">        <span class="nam">fold_ln</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1484" href="#t1484">1484</a></span><span class="t">        <span class="nam">center_writing_weights</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1485" href="#t1485">1485</a></span><span class="t">        <span class="nam">center_unembed</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1486" href="#t1486">1486</a></span><span class="t">        <span class="nam">fold_value_biases</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1487" href="#t1487">1487</a></span><span class="t">        <span class="nam">refactor_factored_attn_matrices</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1488" href="#t1488">1488</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1489" href="#t1489">1489</a></span><span class="t">        <span class="str">"""Load &amp; Process State Dict.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1490" href="#t1490">1490</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1491" href="#t1491">1491</a></span><span class="t"><span class="str">        Load a state dict into the model, and to apply processing to simplify it. The state dict is</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1492" href="#t1492">1492</a></span><span class="t"><span class="str">        assumed to be in the HookedTransformer format.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1461" href="#t1461">1461</a></span><span class="t"><span class="str">        Also, you need to use muAdamW, which rescales the learning rate for output weights and</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1462" href="#t1462">1462</a></span><span class="t"><span class="str">        hidden weights by a factor of 1/fan_in.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1463" href="#t1463">1463</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1464" href="#t1464">1464</a></span><span class="t"><span class="str">        All biases are still assumed to be initialized to 0.0, so we only need to change the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1465" href="#t1465">1465</a></span><span class="t"><span class="str">        weights.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1466" href="#t1466">1466</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1467" href="#t1467">1467</a></span><span class="t">        <span class="key">for</span> <span class="nam">name</span><span class="op">,</span> <span class="nam">param</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">named_parameters</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1468" href="#t1468">1468</a></span><span class="t">            <span class="key">if</span> <span class="str">"W_"</span> <span class="key">in</span> <span class="nam">name</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1469" href="#t1469">1469</a></span><span class="t">                <span class="nam">fan_in</span><span class="op">,</span> <span class="nam">_</span> <span class="op">=</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">calc_fan_in_and_fan_out</span><span class="op">(</span><span class="nam">param</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1470" href="#t1470">1470</a></span><span class="t">                <span class="key">if</span> <span class="str">"embed"</span> <span class="key">in</span> <span class="nam">name</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1471" href="#t1471">1471</a></span><span class="t">                    <span class="nam">scale</span> <span class="op">=</span> <span class="nam">float</span><span class="op">(</span><span class="num">1</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1472" href="#t1472">1472</a></span><span class="t">                <span class="key">elif</span> <span class="str">"unembed"</span> <span class="key">in</span> <span class="nam">name</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1473" href="#t1473">1473</a></span><span class="t">                    <span class="nam">scale</span> <span class="op">=</span> <span class="num">1</span> <span class="op">/</span> <span class="nam">fan_in</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1474" href="#t1474">1474</a></span><span class="t">                <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1475" href="#t1475">1475</a></span><span class="t">                    <span class="nam">scale</span> <span class="op">=</span> <span class="num">1</span> <span class="op">/</span> <span class="nam">fan_in</span><span class="op">**</span><span class="num">0.5</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1476" href="#t1476">1476</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1477" href="#t1477">1477</a></span><span class="t">                <span class="key">if</span> <span class="nam">dist_type</span> <span class="op">==</span> <span class="str">"uniform"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1478" href="#t1478">1478</a></span><span class="t">                    <span class="nam">scale</span> <span class="op">*=</span> <span class="num">3</span><span class="op">**</span><span class="num">0.5</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1479" href="#t1479">1479</a></span><span class="t">                    <span class="nam">nn</span><span class="op">.</span><span class="nam">init</span><span class="op">.</span><span class="nam">uniform_</span><span class="op">(</span><span class="nam">param</span><span class="op">,</span> <span class="op">-</span><span class="nam">scale</span><span class="op">,</span> <span class="nam">scale</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1480" href="#t1480">1480</a></span><span class="t">                <span class="key">elif</span> <span class="nam">dist_type</span> <span class="op">==</span> <span class="str">"normal"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1481" href="#t1481">1481</a></span><span class="t">                    <span class="nam">nn</span><span class="op">.</span><span class="nam">init</span><span class="op">.</span><span class="nam">normal_</span><span class="op">(</span><span class="nam">param</span><span class="op">,</span> <span class="nam">std</span><span class="op">=</span><span class="nam">scale</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1482" href="#t1482">1482</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1483" href="#t1483">1483</a></span><span class="t">    <span class="key">def</span> <span class="nam">load_and_process_state_dict</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1484" href="#t1484">1484</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1485" href="#t1485">1485</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">:</span> <span class="nam">Dict</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1486" href="#t1486">1486</a></span><span class="t">        <span class="nam">fold_ln</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1487" href="#t1487">1487</a></span><span class="t">        <span class="nam">center_writing_weights</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1488" href="#t1488">1488</a></span><span class="t">        <span class="nam">center_unembed</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1489" href="#t1489">1489</a></span><span class="t">        <span class="nam">fold_value_biases</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1490" href="#t1490">1490</a></span><span class="t">        <span class="nam">refactor_factored_attn_matrices</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1491" href="#t1491">1491</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1492" href="#t1492">1492</a></span><span class="t">        <span class="str">"""Load &amp; Process State Dict.</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1493" href="#t1493">1493</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1494" href="#t1494">1494</a></span><span class="t"><span class="str">        See the relevant method (same name as the flag) for more details on the folding, centering</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1495" href="#t1495">1495</a></span><span class="t"><span class="str">        and processing flags.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1494" href="#t1494">1494</a></span><span class="t"><span class="str">        Load a state dict into the model, and to apply processing to simplify it. The state dict is</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1495" href="#t1495">1495</a></span><span class="t"><span class="str">        assumed to be in the HookedTransformer format.</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1496" href="#t1496">1496</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1497" href="#t1497">1497</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1498" href="#t1498">1498</a></span><span class="t"><span class="str">            state_dict (dict): The state dict of the model, in HookedTransformer format. fold_ln</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1499" href="#t1499">1499</a></span><span class="t"><span class="str">            fold_ln (bool, optional): Whether to fold in the LayerNorm weights to the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1500" href="#t1500">1500</a></span><span class="t"><span class="str">                subsequent linear layer. This does not change the computation. Defaults to True.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1501" href="#t1501">1501</a></span><span class="t"><span class="str">            center_writing_weights (bool, optional): Whether to center weights writing to the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1502" href="#t1502">1502</a></span><span class="t"><span class="str">                residual stream (ie set mean to be zero). Due to LayerNorm this doesn't change the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1503" href="#t1503">1503</a></span><span class="t"><span class="str">                computation. Defaults to True.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1504" href="#t1504">1504</a></span><span class="t"><span class="str">            center_unembed (bool, optional): Whether to center W_U (ie set mean to be zero).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1505" href="#t1505">1505</a></span><span class="t"><span class="str">                Softmax is translation invariant so this doesn't affect log probs or loss, but does</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1506" href="#t1506">1506</a></span><span class="t"><span class="str">                change logits. Defaults to True.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1507" href="#t1507">1507</a></span><span class="t"><span class="str">            fold_value_biases (bool, optional): Whether to fold the value biases into the output</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1508" href="#t1508">1508</a></span><span class="t"><span class="str">                bias. Because attention patterns add up to 1, the value biases always have a</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1509" href="#t1509">1509</a></span><span class="t"><span class="str">                constant effect on a layer's output, and it doesn't matter which head a bias is</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1510" href="#t1510">1510</a></span><span class="t"><span class="str">                associated with. We can factor this all into a single output bias to the layer, and</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1511" href="#t1511">1511</a></span><span class="t"><span class="str">                make it easier to interpret the head's output.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1512" href="#t1512">1512</a></span><span class="t"><span class="str">            refactor_factored_attn_matrices (bool, optional): Whether to convert the factored</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1513" href="#t1513">1513</a></span><span class="t"><span class="str">                matrices (W_Q &amp; W_K, and W_O &amp; W_V) to be "even". Defaults to False.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1514" href="#t1514">1514</a></span><span class="t"><span class="str">            model_name (str, optional): checks the model name for special cases of state dict</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1515" href="#t1515">1515</a></span><span class="t"><span class="str">                loading. Only used for Redwood 2L model currently.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1516" href="#t1516">1516</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1517" href="#t1517">1517</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">dtype</span> <span class="key">not</span> <span class="key">in</span> <span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">float32</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">float64</span><span class="op">]</span> <span class="key">and</span> <span class="nam">fold_ln</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1517&#x202F;&#x219B;&#x202F;1518</span><span class="annotate long">line 1517 didn't jump to line 1518, because the condition on line 1517 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1518" href="#t1518">1518</a></span><span class="t">            <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1519" href="#t1519">1519</a></span><span class="t">                <span class="str">"With reduced precision, it is advised to use `from_pretrained_no_processing` instead of `from_pretrained`."</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1520" href="#t1520">1520</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1521" href="#t1521">1521</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1522" href="#t1522">1522</a></span><span class="t">        <span class="key">if</span> <span class="op">(</span>&nbsp;</span><span class="r"><span class="annotate short">1522&#x202F;&#x219B;&#x202F;1527</span><span class="annotate long">line 1522 didn't jump to line 1527</span></span></p>
-    <p class="pln"><span class="n"><a id="t1523" href="#t1523">1523</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">dtype</span> <span class="key">not</span> <span class="key">in</span> <span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">float32</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">float64</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1524" href="#t1524">1524</a></span><span class="t">            <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">num_experts</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1525" href="#t1525">1525</a></span><span class="t">            <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">num_experts</span> <span class="op">></span> <span class="num">1</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1526" href="#t1526">1526</a></span><span class="t">        <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1527" href="#t1527">1527</a></span><span class="t">            <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1528" href="#t1528">1528</a></span><span class="t">                <span class="str">"When running MoE models, it is advised to use a higher precision data type. See docs for more info."</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1529" href="#t1529">1529</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1530" href="#t1530">1530</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1531" href="#t1531">1531</a></span><span class="t">        <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">fill_missing_keys</span><span class="op">(</span><span class="nam">state_dict</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1532" href="#t1532">1532</a></span><span class="t">        <span class="key">if</span> <span class="nam">fold_ln</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1533" href="#t1533">1533</a></span><span class="t">            <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">num_experts</span> <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">num_experts</span> <span class="op">></span> <span class="num">1</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1533&#x202F;&#x219B;&#x202F;1534</span><span class="annotate long">line 1533 didn't jump to line 1534, because the condition on line 1533 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1534" href="#t1534">1534</a></span><span class="t">                <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1535" href="#t1535">1535</a></span><span class="t">                    <span class="str">"You are using MoE, so the layer norm weights can't be folded! Skipping"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1536" href="#t1536">1536</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1537" href="#t1537">1537</a></span><span class="t">            <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="key">in</span> <span class="op">[</span><span class="str">"LN"</span><span class="op">,</span> <span class="str">"LNPre"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1537&#x202F;&#x219B;&#x202F;1539</span><span class="annotate long">line 1537 didn't jump to line 1539, because the condition on line 1537 was never false</span></span></p>
-    <p class="run"><span class="n"><a id="t1538" href="#t1538">1538</a></span><span class="t">                <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">fold_layer_norm</span><span class="op">(</span><span class="nam">state_dict</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1539" href="#t1539">1539</a></span><span class="t">            <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="key">in</span> <span class="op">[</span><span class="str">"RMS"</span><span class="op">,</span> <span class="str">"RMSPre"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1540" href="#t1540">1540</a></span><span class="t">                <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">fold_layer_norm</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1541" href="#t1541">1541</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">,</span> <span class="nam">fold_biases</span><span class="op">=</span><span class="key">False</span><span class="op">,</span> <span class="nam">center_weights</span><span class="op">=</span><span class="key">False</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1542" href="#t1542">1542</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1543" href="#t1543">1543</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1544" href="#t1544">1544</a></span><span class="t">                <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1545" href="#t1545">1545</a></span><span class="t">                    <span class="str">"You are not using LayerNorm or RMSNorm, so the layer norm weights can't be folded! Skipping"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1546" href="#t1546">1546</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1547" href="#t1547">1547</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1548" href="#t1548">1548</a></span><span class="t">        <span class="key">if</span> <span class="nam">center_writing_weights</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1549" href="#t1549">1549</a></span><span class="t">            <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="key">not</span> <span class="key">in</span> <span class="op">[</span><span class="str">"LN"</span><span class="op">,</span> <span class="str">"LNPre"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1549&#x202F;&#x219B;&#x202F;1550</span><span class="annotate long">line 1549 didn't jump to line 1550, because the condition on line 1549 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1550" href="#t1550">1550</a></span><span class="t">                <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1551" href="#t1551">1551</a></span><span class="t">                    <span class="str">"You are not using LayerNorm, so the writing weights can't be centered! Skipping"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1552" href="#t1552">1552</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1553" href="#t1553">1553</a></span><span class="t">            <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">final_rms</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1554" href="#t1554">1554</a></span><span class="t">                <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1555" href="#t1555">1555</a></span><span class="t">                    <span class="str">"This model is using final RMS normalization, so the writing weights can't be centered! Skipping"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1556" href="#t1556">1556</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1557" href="#t1557">1557</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1558" href="#t1558">1558</a></span><span class="t">                <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">center_writing_weights</span><span class="op">(</span><span class="nam">state_dict</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1559" href="#t1559">1559</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1560" href="#t1560">1560</a></span><span class="t">        <span class="key">if</span> <span class="nam">center_unembed</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1561" href="#t1561">1561</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">center_unembed</span><span class="op">(</span><span class="nam">state_dict</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1562" href="#t1562">1562</a></span><span class="t">        <span class="key">if</span> <span class="nam">fold_value_biases</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1563" href="#t1563">1563</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">fold_value_biases</span><span class="op">(</span><span class="nam">state_dict</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1564" href="#t1564">1564</a></span><span class="t">        <span class="key">if</span> <span class="nam">refactor_factored_attn_matrices</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1565" href="#t1565">1565</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">refactor_factored_attn_matrices</span><span class="op">(</span><span class="nam">state_dict</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1566" href="#t1566">1566</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1567" href="#t1567">1567</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">load_in_4bit</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1567&#x202F;&#x219B;&#x202F;1570</span><span class="annotate long">line 1567 didn't jump to line 1570, because the condition on line 1567 was never true</span></span></p>
-    <p class="pln"><span class="n"><a id="t1568" href="#t1568">1568</a></span><span class="t">            <span class="com"># with quantization, parameters should be assigned</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1569" href="#t1569">1569</a></span><span class="t">            <span class="com"># so that quantization settings are not lost</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1570" href="#t1570">1570</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">load_state_dict</span><span class="op">(</span><span class="nam">state_dict</span><span class="op">,</span> <span class="nam">assign</span><span class="op">=</span><span class="key">True</span><span class="op">,</span> <span class="nam">strict</span><span class="op">=</span><span class="key">False</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1571" href="#t1571">1571</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1572" href="#t1572">1572</a></span><span class="t">            <span class="nam">state_dict_keys</span> <span class="op">=</span> <span class="nam">list</span><span class="op">(</span><span class="nam">state_dict</span><span class="op">.</span><span class="nam">keys</span><span class="op">(</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1573" href="#t1573">1573</a></span><span class="t">            <span class="key">for</span> <span class="nam">key</span> <span class="key">in</span> <span class="nam">state_dict_keys</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1574" href="#t1574">1574</a></span><span class="t">                <span class="nam">self</span><span class="op">.</span><span class="nam">load_state_dict</span><span class="op">(</span><span class="op">{</span><span class="nam">key</span><span class="op">:</span> <span class="nam">state_dict</span><span class="op">[</span><span class="nam">key</span><span class="op">]</span><span class="op">}</span><span class="op">,</span> <span class="nam">strict</span><span class="op">=</span><span class="key">False</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1575" href="#t1575">1575</a></span><span class="t">                <span class="key">del</span> <span class="nam">state_dict</span><span class="op">[</span><span class="nam">key</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1576" href="#t1576">1576</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1577" href="#t1577">1577</a></span><span class="t">    <span class="key">def</span> <span class="nam">fill_missing_keys</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">state_dict</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1578" href="#t1578">1578</a></span><span class="t">        <span class="key">return</span> <span class="nam">loading</span><span class="op">.</span><span class="nam">fill_missing_keys</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">state_dict</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1497" href="#t1497">1497</a></span><span class="t"><span class="str">        See the relevant method (same name as the flag) for more details on the folding, centering</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1498" href="#t1498">1498</a></span><span class="t"><span class="str">        and processing flags.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1499" href="#t1499">1499</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1500" href="#t1500">1500</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1501" href="#t1501">1501</a></span><span class="t"><span class="str">            state_dict (dict): The state dict of the model, in HookedTransformer format. fold_ln</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1502" href="#t1502">1502</a></span><span class="t"><span class="str">            fold_ln (bool, optional): Whether to fold in the LayerNorm weights to the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1503" href="#t1503">1503</a></span><span class="t"><span class="str">                subsequent linear layer. This does not change the computation. Defaults to True.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1504" href="#t1504">1504</a></span><span class="t"><span class="str">            center_writing_weights (bool, optional): Whether to center weights writing to the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1505" href="#t1505">1505</a></span><span class="t"><span class="str">                residual stream (ie set mean to be zero). Due to LayerNorm this doesn't change the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1506" href="#t1506">1506</a></span><span class="t"><span class="str">                computation. Defaults to True.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1507" href="#t1507">1507</a></span><span class="t"><span class="str">            center_unembed (bool, optional): Whether to center W_U (ie set mean to be zero).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1508" href="#t1508">1508</a></span><span class="t"><span class="str">                Softmax is translation invariant so this doesn't affect log probs or loss, but does</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1509" href="#t1509">1509</a></span><span class="t"><span class="str">                change logits. Defaults to True.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1510" href="#t1510">1510</a></span><span class="t"><span class="str">            fold_value_biases (bool, optional): Whether to fold the value biases into the output</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1511" href="#t1511">1511</a></span><span class="t"><span class="str">                bias. Because attention patterns add up to 1, the value biases always have a</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1512" href="#t1512">1512</a></span><span class="t"><span class="str">                constant effect on a layer's output, and it doesn't matter which head a bias is</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1513" href="#t1513">1513</a></span><span class="t"><span class="str">                associated with. We can factor this all into a single output bias to the layer, and</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1514" href="#t1514">1514</a></span><span class="t"><span class="str">                make it easier to interpret the head's output.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1515" href="#t1515">1515</a></span><span class="t"><span class="str">            refactor_factored_attn_matrices (bool, optional): Whether to convert the factored</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1516" href="#t1516">1516</a></span><span class="t"><span class="str">                matrices (W_Q &amp; W_K, and W_O &amp; W_V) to be "even". Defaults to False.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1517" href="#t1517">1517</a></span><span class="t"><span class="str">            model_name (str, optional): checks the model name for special cases of state dict</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1518" href="#t1518">1518</a></span><span class="t"><span class="str">                loading. Only used for Redwood 2L model currently.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1519" href="#t1519">1519</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1520" href="#t1520">1520</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">dtype</span> <span class="key">not</span> <span class="key">in</span> <span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">float32</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">float64</span><span class="op">]</span> <span class="key">and</span> <span class="nam">fold_ln</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1520&#x202F;&#x219B;&#x202F;1521</span><span class="annotate long">line 1520 didn't jump to line 1521, because the condition on line 1520 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1521" href="#t1521">1521</a></span><span class="t">            <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1522" href="#t1522">1522</a></span><span class="t">                <span class="str">"With reduced precision, it is advised to use `from_pretrained_no_processing` instead of `from_pretrained`."</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1523" href="#t1523">1523</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1524" href="#t1524">1524</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1525" href="#t1525">1525</a></span><span class="t">        <span class="key">if</span> <span class="op">(</span>&nbsp;</span><span class="r"><span class="annotate short">1525&#x202F;&#x219B;&#x202F;1530</span><span class="annotate long">line 1525 didn't jump to line 1530</span></span></p>
+    <p class="pln"><span class="n"><a id="t1526" href="#t1526">1526</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">dtype</span> <span class="key">not</span> <span class="key">in</span> <span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">float32</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">float64</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1527" href="#t1527">1527</a></span><span class="t">            <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">num_experts</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1528" href="#t1528">1528</a></span><span class="t">            <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">num_experts</span> <span class="op">></span> <span class="num">1</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1529" href="#t1529">1529</a></span><span class="t">        <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1530" href="#t1530">1530</a></span><span class="t">            <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1531" href="#t1531">1531</a></span><span class="t">                <span class="str">"When running MoE models, it is advised to use a higher precision data type. See docs for more info."</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1532" href="#t1532">1532</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1533" href="#t1533">1533</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1534" href="#t1534">1534</a></span><span class="t">        <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">fill_missing_keys</span><span class="op">(</span><span class="nam">state_dict</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1535" href="#t1535">1535</a></span><span class="t">        <span class="key">if</span> <span class="nam">fold_ln</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1536" href="#t1536">1536</a></span><span class="t">            <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">num_experts</span> <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">num_experts</span> <span class="op">></span> <span class="num">1</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1536&#x202F;&#x219B;&#x202F;1537</span><span class="annotate long">line 1536 didn't jump to line 1537, because the condition on line 1536 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1537" href="#t1537">1537</a></span><span class="t">                <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1538" href="#t1538">1538</a></span><span class="t">                    <span class="str">"You are using MoE, so the layer norm weights can't be folded! Skipping"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1539" href="#t1539">1539</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1540" href="#t1540">1540</a></span><span class="t">            <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="key">in</span> <span class="op">[</span><span class="str">"LN"</span><span class="op">,</span> <span class="str">"LNPre"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1540&#x202F;&#x219B;&#x202F;1542</span><span class="annotate long">line 1540 didn't jump to line 1542, because the condition on line 1540 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t1541" href="#t1541">1541</a></span><span class="t">                <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">fold_layer_norm</span><span class="op">(</span><span class="nam">state_dict</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1542" href="#t1542">1542</a></span><span class="t">            <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="key">in</span> <span class="op">[</span><span class="str">"RMS"</span><span class="op">,</span> <span class="str">"RMSPre"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1543" href="#t1543">1543</a></span><span class="t">                <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">fold_layer_norm</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1544" href="#t1544">1544</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">,</span> <span class="nam">fold_biases</span><span class="op">=</span><span class="key">False</span><span class="op">,</span> <span class="nam">center_weights</span><span class="op">=</span><span class="key">False</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1545" href="#t1545">1545</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1546" href="#t1546">1546</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1547" href="#t1547">1547</a></span><span class="t">                <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1548" href="#t1548">1548</a></span><span class="t">                    <span class="str">"You are not using LayerNorm or RMSNorm, so the layer norm weights can't be folded! Skipping"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1549" href="#t1549">1549</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1550" href="#t1550">1550</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1551" href="#t1551">1551</a></span><span class="t">        <span class="key">if</span> <span class="nam">center_writing_weights</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1552" href="#t1552">1552</a></span><span class="t">            <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="key">not</span> <span class="key">in</span> <span class="op">[</span><span class="str">"LN"</span><span class="op">,</span> <span class="str">"LNPre"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1552&#x202F;&#x219B;&#x202F;1553</span><span class="annotate long">line 1552 didn't jump to line 1553, because the condition on line 1552 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1553" href="#t1553">1553</a></span><span class="t">                <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1554" href="#t1554">1554</a></span><span class="t">                    <span class="str">"You are not using LayerNorm, so the writing weights can't be centered! Skipping"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1555" href="#t1555">1555</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1556" href="#t1556">1556</a></span><span class="t">            <span class="key">elif</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">final_rms</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1557" href="#t1557">1557</a></span><span class="t">                <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1558" href="#t1558">1558</a></span><span class="t">                    <span class="str">"This model is using final RMS normalization, so the writing weights can't be centered! Skipping"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1559" href="#t1559">1559</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1560" href="#t1560">1560</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1561" href="#t1561">1561</a></span><span class="t">                <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">center_writing_weights</span><span class="op">(</span><span class="nam">state_dict</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1562" href="#t1562">1562</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1563" href="#t1563">1563</a></span><span class="t">        <span class="key">if</span> <span class="nam">center_unembed</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1564" href="#t1564">1564</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">center_unembed</span><span class="op">(</span><span class="nam">state_dict</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1565" href="#t1565">1565</a></span><span class="t">        <span class="key">if</span> <span class="nam">fold_value_biases</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1566" href="#t1566">1566</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">fold_value_biases</span><span class="op">(</span><span class="nam">state_dict</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1567" href="#t1567">1567</a></span><span class="t">        <span class="key">if</span> <span class="nam">refactor_factored_attn_matrices</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1568" href="#t1568">1568</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">refactor_factored_attn_matrices</span><span class="op">(</span><span class="nam">state_dict</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1569" href="#t1569">1569</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1570" href="#t1570">1570</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">load_in_4bit</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1570&#x202F;&#x219B;&#x202F;1573</span><span class="annotate long">line 1570 didn't jump to line 1573, because the condition on line 1570 was never true</span></span></p>
+    <p class="pln"><span class="n"><a id="t1571" href="#t1571">1571</a></span><span class="t">            <span class="com"># with quantization, parameters should be assigned</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1572" href="#t1572">1572</a></span><span class="t">            <span class="com"># so that quantization settings are not lost</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1573" href="#t1573">1573</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">load_state_dict</span><span class="op">(</span><span class="nam">state_dict</span><span class="op">,</span> <span class="nam">assign</span><span class="op">=</span><span class="key">True</span><span class="op">,</span> <span class="nam">strict</span><span class="op">=</span><span class="key">False</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1574" href="#t1574">1574</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1575" href="#t1575">1575</a></span><span class="t">            <span class="nam">state_dict_keys</span> <span class="op">=</span> <span class="nam">list</span><span class="op">(</span><span class="nam">state_dict</span><span class="op">.</span><span class="nam">keys</span><span class="op">(</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1576" href="#t1576">1576</a></span><span class="t">            <span class="key">for</span> <span class="nam">key</span> <span class="key">in</span> <span class="nam">state_dict_keys</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1577" href="#t1577">1577</a></span><span class="t">                <span class="nam">self</span><span class="op">.</span><span class="nam">load_state_dict</span><span class="op">(</span><span class="op">{</span><span class="nam">key</span><span class="op">:</span> <span class="nam">state_dict</span><span class="op">[</span><span class="nam">key</span><span class="op">]</span><span class="op">}</span><span class="op">,</span> <span class="nam">strict</span><span class="op">=</span><span class="key">False</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1578" href="#t1578">1578</a></span><span class="t">                <span class="key">del</span> <span class="nam">state_dict</span><span class="op">[</span><span class="nam">key</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1579" href="#t1579">1579</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1580" href="#t1580">1580</a></span><span class="t">    <span class="key">def</span> <span class="nam">fold_layer_norm</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1581" href="#t1581">1581</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span> <span class="nam">state_dict</span><span class="op">:</span> <span class="nam">Dict</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">]</span><span class="op">,</span> <span class="nam">fold_biases</span><span class="op">=</span><span class="key">True</span><span class="op">,</span> <span class="nam">center_weights</span><span class="op">=</span><span class="key">True</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1582" href="#t1582">1582</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1583" href="#t1583">1583</a></span><span class="t">        <span class="str">"""Fold Layer Norm. Can also be used to fold RMS Norm, when fold_biases and center_weights are set to False.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1584" href="#t1584">1584</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1585" href="#t1585">1585</a></span><span class="t"><span class="str">        Takes in a state dict from a pretrained model, formatted to be consistent with</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1586" href="#t1586">1586</a></span><span class="t"><span class="str">        HookedTransformer but with LayerNorm weights and biases. Folds these into the neighbouring</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1587" href="#t1587">1587</a></span><span class="t"><span class="str">        weights. See further_comments.md for more details.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1588" href="#t1588">1588</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1589" href="#t1589">1589</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1590" href="#t1590">1590</a></span><span class="t"><span class="str">            state_dict (Dict[str, torch.Tensor]): State dict of pretrained model.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1591" href="#t1591">1591</a></span><span class="t"><span class="str">            fold_biases (bool): Enables folding of LN biases. Should be disabled when RMS Norm is used.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1592" href="#t1592">1592</a></span><span class="t"><span class="str">            center_weights (bool): Enables the centering of weights after folding in LN. Should be disabled when RMS Norm is used.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1593" href="#t1593">1593</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1594" href="#t1594">1594</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1595" href="#t1595">1595</a></span><span class="t">        <span class="com"># Models that use Grouped Query Attention (Only Mistral at the time of writing) prefix their K/V weights and</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1596" href="#t1596">1596</a></span><span class="t">        <span class="com"># biases with an underscore in order to distinguish them, but folding the LN into them still works the same,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1597" href="#t1597">1597</a></span><span class="t">        <span class="com"># so we just add the underscore if GQA is used (i.e. if `cfg.n_key_value_heads is specified`).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1598" href="#t1598">1598</a></span><span class="t">        <span class="nam">gqa</span> <span class="op">=</span> <span class="str">""</span> <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_key_value_heads</span> <span class="key">is</span> <span class="key">None</span> <span class="key">else</span> <span class="str">"_"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1599" href="#t1599">1599</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1600" href="#t1600">1600</a></span><span class="t">        <span class="key">for</span> <span class="nam">l</span> <span class="key">in</span> <span class="nam">range</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1601" href="#t1601">1601</a></span><span class="t">            <span class="com"># Fold ln1 into attention - it's important to fold biases first, since biases depend on</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1602" href="#t1602">1602</a></span><span class="t">            <span class="com"># weights but not vice versa The various indexing is just to broadcast ln.b and ln.w</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1603" href="#t1603">1603</a></span><span class="t">            <span class="com"># along every axis other than d_model. Each weight matrix right multiplies. To fold in</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1604" href="#t1604">1604</a></span><span class="t">            <span class="com"># the bias, we use the W_ matrix to map it to the hidden space of the layer, so we need</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1605" href="#t1605">1605</a></span><span class="t">            <span class="com"># to sum along axis -2, which is the residual stream space axis.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1606" href="#t1606">1606</a></span><span class="t">            <span class="key">if</span> <span class="nam">fold_biases</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1606&#x202F;&#x219B;&#x202F;1629</span><span class="annotate long">line 1606 didn't jump to line 1629</span></span></p>
-    <p class="run"><span class="n"><a id="t1607" href="#t1607">1607</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_Q"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_Q"</span><span class="op">]</span> <span class="op">+</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1608" href="#t1608">1608</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_Q"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1609" href="#t1609">1609</a></span><span class="t">                    <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln1.b"</span><span class="op">]</span><span class="op">[</span><span class="key">None</span><span class="op">,</span> <span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1610" href="#t1610">1610</a></span><span class="t">                <span class="op">)</span><span class="op">.</span><span class="nam">sum</span><span class="op">(</span><span class="op">-</span><span class="num">2</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1611" href="#t1611">1611</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.{gqa}b_K"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1612" href="#t1612">1612</a></span><span class="t">                    <span class="str">f"blocks.{l}.attn.{gqa}b_K"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1613" href="#t1613">1613</a></span><span class="t">                <span class="op">]</span> <span class="op">+</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1614" href="#t1614">1614</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.{gqa}W_K"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1615" href="#t1615">1615</a></span><span class="t">                    <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln1.b"</span><span class="op">]</span><span class="op">[</span><span class="key">None</span><span class="op">,</span> <span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1616" href="#t1616">1616</a></span><span class="t">                <span class="op">)</span><span class="op">.</span><span class="nam">sum</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1617" href="#t1617">1617</a></span><span class="t">                    <span class="op">-</span><span class="num">2</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1618" href="#t1618">1618</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1619" href="#t1619">1619</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.{gqa}b_V"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1620" href="#t1620">1620</a></span><span class="t">                    <span class="str">f"blocks.{l}.attn.{gqa}b_V"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1621" href="#t1621">1621</a></span><span class="t">                <span class="op">]</span> <span class="op">+</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1622" href="#t1622">1622</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.{gqa}W_V"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1623" href="#t1623">1623</a></span><span class="t">                    <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln1.b"</span><span class="op">]</span><span class="op">[</span><span class="key">None</span><span class="op">,</span> <span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1624" href="#t1624">1624</a></span><span class="t">                <span class="op">)</span><span class="op">.</span><span class="nam">sum</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1625" href="#t1625">1625</a></span><span class="t">                    <span class="op">-</span><span class="num">2</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1626" href="#t1626">1626</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1627" href="#t1627">1627</a></span><span class="t">                <span class="key">del</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln1.b"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1628" href="#t1628">1628</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1629" href="#t1629">1629</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_Q"</span><span class="op">]</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1630" href="#t1630">1630</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_Q"</span><span class="op">]</span> <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln1.w"</span><span class="op">]</span><span class="op">[</span><span class="key">None</span><span class="op">,</span> <span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1631" href="#t1631">1631</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1632" href="#t1632">1632</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.{gqa}W_K"</span><span class="op">]</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1633" href="#t1633">1633</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.{gqa}W_K"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1634" href="#t1634">1634</a></span><span class="t">                <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln1.w"</span><span class="op">]</span><span class="op">[</span><span class="key">None</span><span class="op">,</span> <span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1635" href="#t1635">1635</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1636" href="#t1636">1636</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.{gqa}W_V"</span><span class="op">]</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1637" href="#t1637">1637</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.{gqa}W_V"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1638" href="#t1638">1638</a></span><span class="t">                <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln1.w"</span><span class="op">]</span><span class="op">[</span><span class="key">None</span><span class="op">,</span> <span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1639" href="#t1639">1639</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1640" href="#t1640">1640</a></span><span class="t">            <span class="key">del</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln1.w"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1641" href="#t1641">1641</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1642" href="#t1642">1642</a></span><span class="t">            <span class="com"># Finally, we center the weights reading from the residual stream. The output of the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1643" href="#t1643">1643</a></span><span class="t">            <span class="com"># first part of the LayerNorm is mean 0 and standard deviation 1, so the mean of any</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1644" href="#t1644">1644</a></span><span class="t">            <span class="com"># input vector of the matrix doesn't matter and can be set to zero. Equivalently, the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1645" href="#t1645">1645</a></span><span class="t">            <span class="com"># output of LayerNormPre is orthogonal to the vector of all 1s (because dotting with</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1646" href="#t1646">1646</a></span><span class="t">            <span class="com"># that gets the sum), so we can remove the component of the matrix parallel to this.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1647" href="#t1647">1647</a></span><span class="t">            <span class="key">if</span> <span class="nam">center_weights</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1647&#x202F;&#x219B;&#x202F;1665</span><span class="annotate long">line 1647 didn't jump to line 1665, because the condition on line 1647 was never false</span></span></p>
-    <p class="run"><span class="n"><a id="t1648" href="#t1648">1648</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_Q"</span><span class="op">]</span> <span class="op">-=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">reduce</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1649" href="#t1649">1649</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_Q"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1650" href="#t1650">1650</a></span><span class="t">                    <span class="str">"head_index d_model d_head -> head_index 1 d_head"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1651" href="#t1651">1651</a></span><span class="t">                    <span class="str">"mean"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1652" href="#t1652">1652</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1653" href="#t1653">1653</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.{gqa}W_K"</span><span class="op">]</span> <span class="op">-=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">reduce</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1654" href="#t1654">1654</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.{gqa}W_K"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1655" href="#t1655">1655</a></span><span class="t">                    <span class="str">"head_index d_model d_head -> head_index 1 d_head"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1656" href="#t1656">1656</a></span><span class="t">                    <span class="str">"mean"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1657" href="#t1657">1657</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1658" href="#t1658">1658</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.{gqa}W_V"</span><span class="op">]</span> <span class="op">-=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">reduce</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1659" href="#t1659">1659</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.{gqa}W_V"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1660" href="#t1660">1660</a></span><span class="t">                    <span class="str">"head_index d_model d_head -> head_index 1 d_head"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1661" href="#t1661">1661</a></span><span class="t">                    <span class="str">"mean"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1662" href="#t1662">1662</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1663" href="#t1663">1663</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1664" href="#t1664">1664</a></span><span class="t">            <span class="com"># Fold ln2 into MLP</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1665" href="#t1665">1665</a></span><span class="t">            <span class="key">if</span> <span class="key">not</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">attn_only</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1666" href="#t1666">1666</a></span><span class="t">                <span class="key">if</span> <span class="nam">fold_biases</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1666&#x202F;&#x219B;&#x202F;1673</span><span class="annotate long">line 1666 didn't jump to line 1673</span></span></p>
-    <p class="run"><span class="n"><a id="t1667" href="#t1667">1667</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.b_in"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.b_in"</span><span class="op">]</span> <span class="op">+</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1668" href="#t1668">1668</a></span><span class="t">                        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_in"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1669" href="#t1669">1669</a></span><span class="t">                        <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln2.b"</span><span class="op">]</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1670" href="#t1670">1670</a></span><span class="t">                    <span class="op">)</span><span class="op">.</span><span class="nam">sum</span><span class="op">(</span><span class="op">-</span><span class="num">2</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1671" href="#t1671">1671</a></span><span class="t">                    <span class="key">del</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln2.b"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1672" href="#t1672">1672</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1673" href="#t1673">1673</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_in"</span><span class="op">]</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1674" href="#t1674">1674</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_in"</span><span class="op">]</span> <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln2.w"</span><span class="op">]</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1675" href="#t1675">1675</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1676" href="#t1676">1676</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1677" href="#t1677">1677</a></span><span class="t">                <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">gated_mlp</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1677&#x202F;&#x219B;&#x202F;1678</span><span class="annotate long">line 1677 didn't jump to line 1678</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1678" href="#t1678">1678</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_gate"</span><span class="op">]</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1679" href="#t1679">1679</a></span><span class="t">                        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_gate"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1680" href="#t1680">1680</a></span><span class="t">                        <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln2.w"</span><span class="op">]</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1681" href="#t1681">1681</a></span><span class="t">                    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1682" href="#t1682">1682</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1683" href="#t1683">1683</a></span><span class="t">                <span class="key">del</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln2.w"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1684" href="#t1684">1684</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1685" href="#t1685">1685</a></span><span class="t">                <span class="key">if</span> <span class="nam">center_weights</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1685&#x202F;&#x219B;&#x202F;1693</span><span class="annotate long">line 1685 didn't jump to line 1693, because the condition on line 1685 was never false</span></span></p>
-    <p class="pln"><span class="n"><a id="t1686" href="#t1686">1686</a></span><span class="t">                    <span class="com"># Center the weights that read in from the LayerNormPre</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1687" href="#t1687">1687</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_in"</span><span class="op">]</span> <span class="op">-=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">reduce</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1688" href="#t1688">1688</a></span><span class="t">                        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_in"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1689" href="#t1689">1689</a></span><span class="t">                        <span class="str">"d_model d_mlp -> 1 d_mlp"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1690" href="#t1690">1690</a></span><span class="t">                        <span class="str">"mean"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1691" href="#t1691">1691</a></span><span class="t">                    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1692" href="#t1692">1692</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1693" href="#t1693">1693</a></span><span class="t">                <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">act_fn</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span> <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">act_fn</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"solu"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1694" href="#t1694">1694</a></span><span class="t">                    <span class="com"># Fold ln3 into activation</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1695" href="#t1695">1695</a></span><span class="t">                    <span class="key">if</span> <span class="nam">fold_biases</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1695&#x202F;&#x219B;&#x202F;1707</span><span class="annotate long">line 1695 didn't jump to line 1707</span></span></p>
-    <p class="run"><span class="n"><a id="t1696" href="#t1696">1696</a></span><span class="t">                        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.b_out"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1697" href="#t1697">1697</a></span><span class="t">                            <span class="str">f"blocks.{l}.mlp.b_out"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1698" href="#t1698">1698</a></span><span class="t">                        <span class="op">]</span> <span class="op">+</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1699" href="#t1699">1699</a></span><span class="t">                            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_out"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1700" href="#t1700">1700</a></span><span class="t">                            <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.ln.b"</span><span class="op">]</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1701" href="#t1701">1701</a></span><span class="t">                        <span class="op">)</span><span class="op">.</span><span class="nam">sum</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1702" href="#t1702">1702</a></span><span class="t">                            <span class="op">-</span><span class="num">2</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1703" href="#t1703">1703</a></span><span class="t">                        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1704" href="#t1704">1704</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1705" href="#t1705">1705</a></span><span class="t">                        <span class="key">del</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.ln.b"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1706" href="#t1706">1706</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1707" href="#t1707">1707</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_out"</span><span class="op">]</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1708" href="#t1708">1708</a></span><span class="t">                        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_out"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1709" href="#t1709">1709</a></span><span class="t">                        <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.ln.w"</span><span class="op">]</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1710" href="#t1710">1710</a></span><span class="t">                    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1711" href="#t1711">1711</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1712" href="#t1712">1712</a></span><span class="t">                    <span class="key">if</span> <span class="nam">center_weights</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1712&#x202F;&#x219B;&#x202F;1720</span><span class="annotate long">line 1712 didn't jump to line 1720, because the condition on line 1712 was never false</span></span></p>
-    <p class="pln"><span class="n"><a id="t1713" href="#t1713">1713</a></span><span class="t">                        <span class="com"># Center the weights that read in from the LayerNormPre</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1714" href="#t1714">1714</a></span><span class="t">                        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_out"</span><span class="op">]</span> <span class="op">-=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">reduce</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1715" href="#t1715">1715</a></span><span class="t">                            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_out"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1716" href="#t1716">1716</a></span><span class="t">                            <span class="str">"d_mlp d_model -> 1 d_model"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1717" href="#t1717">1717</a></span><span class="t">                            <span class="str">"mean"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1718" href="#t1718">1718</a></span><span class="t">                        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1719" href="#t1719">1719</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1720" href="#t1720">1720</a></span><span class="t">                    <span class="key">del</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.ln.w"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1721" href="#t1721">1721</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1722" href="#t1722">1722</a></span><span class="t">        <span class="com"># Fold ln_final into Unembed</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1723" href="#t1723">1723</a></span><span class="t">        <span class="key">if</span> <span class="key">not</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">final_rms</span> <span class="key">and</span> <span class="nam">fold_biases</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1724" href="#t1724">1724</a></span><span class="t">            <span class="com"># Dumb bug from my old SoLU training code, some models have RMSNorm instead of LayerNorm</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1725" href="#t1725">1725</a></span><span class="t">            <span class="com"># pre unembed.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1726" href="#t1726">1726</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"unembed.b_U"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"unembed.b_U"</span><span class="op">]</span> <span class="op">+</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1727" href="#t1727">1727</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"unembed.W_U"</span><span class="op">]</span> <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"ln_final.b"</span><span class="op">]</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1728" href="#t1728">1728</a></span><span class="t">            <span class="op">)</span><span class="op">.</span><span class="nam">sum</span><span class="op">(</span><span class="nam">dim</span><span class="op">=</span><span class="op">-</span><span class="num">2</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1729" href="#t1729">1729</a></span><span class="t">            <span class="key">del</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"ln_final.b"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1730" href="#t1730">1730</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1731" href="#t1731">1731</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"unembed.W_U"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"unembed.W_U"</span><span class="op">]</span> <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"ln_final.w"</span><span class="op">]</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1732" href="#t1732">1732</a></span><span class="t">        <span class="key">del</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"ln_final.w"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1580" href="#t1580">1580</a></span><span class="t">    <span class="key">def</span> <span class="nam">fill_missing_keys</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">state_dict</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1581" href="#t1581">1581</a></span><span class="t">        <span class="key">return</span> <span class="nam">loading</span><span class="op">.</span><span class="nam">fill_missing_keys</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">state_dict</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1582" href="#t1582">1582</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1583" href="#t1583">1583</a></span><span class="t">    <span class="key">def</span> <span class="nam">fold_layer_norm</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1584" href="#t1584">1584</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span> <span class="nam">state_dict</span><span class="op">:</span> <span class="nam">Dict</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">]</span><span class="op">,</span> <span class="nam">fold_biases</span><span class="op">=</span><span class="key">True</span><span class="op">,</span> <span class="nam">center_weights</span><span class="op">=</span><span class="key">True</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1585" href="#t1585">1585</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1586" href="#t1586">1586</a></span><span class="t">        <span class="str">"""Fold Layer Norm. Can also be used to fold RMS Norm, when fold_biases and center_weights are set to False.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1587" href="#t1587">1587</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1588" href="#t1588">1588</a></span><span class="t"><span class="str">        Takes in a state dict from a pretrained model, formatted to be consistent with</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1589" href="#t1589">1589</a></span><span class="t"><span class="str">        HookedTransformer but with LayerNorm weights and biases. Folds these into the neighbouring</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1590" href="#t1590">1590</a></span><span class="t"><span class="str">        weights. See further_comments.md for more details.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1591" href="#t1591">1591</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1592" href="#t1592">1592</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1593" href="#t1593">1593</a></span><span class="t"><span class="str">            state_dict (Dict[str, torch.Tensor]): State dict of pretrained model.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1594" href="#t1594">1594</a></span><span class="t"><span class="str">            fold_biases (bool): Enables folding of LN biases. Should be disabled when RMS Norm is used.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1595" href="#t1595">1595</a></span><span class="t"><span class="str">            center_weights (bool): Enables the centering of weights after folding in LN. Should be disabled when RMS Norm is used.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1596" href="#t1596">1596</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1597" href="#t1597">1597</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1598" href="#t1598">1598</a></span><span class="t">        <span class="com"># Models that use Grouped Query Attention (Only Mistral at the time of writing) prefix their K/V weights and</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1599" href="#t1599">1599</a></span><span class="t">        <span class="com"># biases with an underscore in order to distinguish them, but folding the LN into them still works the same,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1600" href="#t1600">1600</a></span><span class="t">        <span class="com"># so we just add the underscore if GQA is used (i.e. if `cfg.n_key_value_heads is specified`).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1601" href="#t1601">1601</a></span><span class="t">        <span class="nam">gqa</span> <span class="op">=</span> <span class="str">""</span> <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_key_value_heads</span> <span class="key">is</span> <span class="key">None</span> <span class="key">else</span> <span class="str">"_"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1602" href="#t1602">1602</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1603" href="#t1603">1603</a></span><span class="t">        <span class="key">for</span> <span class="nam">l</span> <span class="key">in</span> <span class="nam">range</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1604" href="#t1604">1604</a></span><span class="t">            <span class="com"># Fold ln1 into attention - it's important to fold biases first, since biases depend on</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1605" href="#t1605">1605</a></span><span class="t">            <span class="com"># weights but not vice versa The various indexing is just to broadcast ln.b and ln.w</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1606" href="#t1606">1606</a></span><span class="t">            <span class="com"># along every axis other than d_model. Each weight matrix right multiplies. To fold in</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1607" href="#t1607">1607</a></span><span class="t">            <span class="com"># the bias, we use the W_ matrix to map it to the hidden space of the layer, so we need</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1608" href="#t1608">1608</a></span><span class="t">            <span class="com"># to sum along axis -2, which is the residual stream space axis.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1609" href="#t1609">1609</a></span><span class="t">            <span class="key">if</span> <span class="nam">fold_biases</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1609&#x202F;&#x219B;&#x202F;1632</span><span class="annotate long">line 1609 didn't jump to line 1632</span></span></p>
+    <p class="run"><span class="n"><a id="t1610" href="#t1610">1610</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_Q"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_Q"</span><span class="op">]</span> <span class="op">+</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1611" href="#t1611">1611</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_Q"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1612" href="#t1612">1612</a></span><span class="t">                    <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln1.b"</span><span class="op">]</span><span class="op">[</span><span class="key">None</span><span class="op">,</span> <span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1613" href="#t1613">1613</a></span><span class="t">                <span class="op">)</span><span class="op">.</span><span class="nam">sum</span><span class="op">(</span><span class="op">-</span><span class="num">2</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1614" href="#t1614">1614</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.{gqa}b_K"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1615" href="#t1615">1615</a></span><span class="t">                    <span class="str">f"blocks.{l}.attn.{gqa}b_K"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1616" href="#t1616">1616</a></span><span class="t">                <span class="op">]</span> <span class="op">+</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1617" href="#t1617">1617</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.{gqa}W_K"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1618" href="#t1618">1618</a></span><span class="t">                    <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln1.b"</span><span class="op">]</span><span class="op">[</span><span class="key">None</span><span class="op">,</span> <span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1619" href="#t1619">1619</a></span><span class="t">                <span class="op">)</span><span class="op">.</span><span class="nam">sum</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1620" href="#t1620">1620</a></span><span class="t">                    <span class="op">-</span><span class="num">2</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1621" href="#t1621">1621</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1622" href="#t1622">1622</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.{gqa}b_V"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1623" href="#t1623">1623</a></span><span class="t">                    <span class="str">f"blocks.{l}.attn.{gqa}b_V"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1624" href="#t1624">1624</a></span><span class="t">                <span class="op">]</span> <span class="op">+</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1625" href="#t1625">1625</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.{gqa}W_V"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1626" href="#t1626">1626</a></span><span class="t">                    <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln1.b"</span><span class="op">]</span><span class="op">[</span><span class="key">None</span><span class="op">,</span> <span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1627" href="#t1627">1627</a></span><span class="t">                <span class="op">)</span><span class="op">.</span><span class="nam">sum</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1628" href="#t1628">1628</a></span><span class="t">                    <span class="op">-</span><span class="num">2</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1629" href="#t1629">1629</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1630" href="#t1630">1630</a></span><span class="t">                <span class="key">del</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln1.b"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1631" href="#t1631">1631</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1632" href="#t1632">1632</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_Q"</span><span class="op">]</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1633" href="#t1633">1633</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_Q"</span><span class="op">]</span> <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln1.w"</span><span class="op">]</span><span class="op">[</span><span class="key">None</span><span class="op">,</span> <span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1634" href="#t1634">1634</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1635" href="#t1635">1635</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.{gqa}W_K"</span><span class="op">]</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1636" href="#t1636">1636</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.{gqa}W_K"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1637" href="#t1637">1637</a></span><span class="t">                <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln1.w"</span><span class="op">]</span><span class="op">[</span><span class="key">None</span><span class="op">,</span> <span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1638" href="#t1638">1638</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1639" href="#t1639">1639</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.{gqa}W_V"</span><span class="op">]</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1640" href="#t1640">1640</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.{gqa}W_V"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1641" href="#t1641">1641</a></span><span class="t">                <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln1.w"</span><span class="op">]</span><span class="op">[</span><span class="key">None</span><span class="op">,</span> <span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1642" href="#t1642">1642</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1643" href="#t1643">1643</a></span><span class="t">            <span class="key">del</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln1.w"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1644" href="#t1644">1644</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1645" href="#t1645">1645</a></span><span class="t">            <span class="com"># Finally, we center the weights reading from the residual stream. The output of the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1646" href="#t1646">1646</a></span><span class="t">            <span class="com"># first part of the LayerNorm is mean 0 and standard deviation 1, so the mean of any</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1647" href="#t1647">1647</a></span><span class="t">            <span class="com"># input vector of the matrix doesn't matter and can be set to zero. Equivalently, the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1648" href="#t1648">1648</a></span><span class="t">            <span class="com"># output of LayerNormPre is orthogonal to the vector of all 1s (because dotting with</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1649" href="#t1649">1649</a></span><span class="t">            <span class="com"># that gets the sum), so we can remove the component of the matrix parallel to this.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1650" href="#t1650">1650</a></span><span class="t">            <span class="key">if</span> <span class="nam">center_weights</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1650&#x202F;&#x219B;&#x202F;1668</span><span class="annotate long">line 1650 didn't jump to line 1668, because the condition on line 1650 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t1651" href="#t1651">1651</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_Q"</span><span class="op">]</span> <span class="op">-=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">reduce</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1652" href="#t1652">1652</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_Q"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1653" href="#t1653">1653</a></span><span class="t">                    <span class="str">"head_index d_model d_head -> head_index 1 d_head"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1654" href="#t1654">1654</a></span><span class="t">                    <span class="str">"mean"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1655" href="#t1655">1655</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1656" href="#t1656">1656</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.{gqa}W_K"</span><span class="op">]</span> <span class="op">-=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">reduce</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1657" href="#t1657">1657</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.{gqa}W_K"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1658" href="#t1658">1658</a></span><span class="t">                    <span class="str">"head_index d_model d_head -> head_index 1 d_head"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1659" href="#t1659">1659</a></span><span class="t">                    <span class="str">"mean"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1660" href="#t1660">1660</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1661" href="#t1661">1661</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.{gqa}W_V"</span><span class="op">]</span> <span class="op">-=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">reduce</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1662" href="#t1662">1662</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.{gqa}W_V"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1663" href="#t1663">1663</a></span><span class="t">                    <span class="str">"head_index d_model d_head -> head_index 1 d_head"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1664" href="#t1664">1664</a></span><span class="t">                    <span class="str">"mean"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1665" href="#t1665">1665</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1666" href="#t1666">1666</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1667" href="#t1667">1667</a></span><span class="t">            <span class="com"># Fold ln2 into MLP</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1668" href="#t1668">1668</a></span><span class="t">            <span class="key">if</span> <span class="key">not</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">attn_only</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1669" href="#t1669">1669</a></span><span class="t">                <span class="key">if</span> <span class="nam">fold_biases</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1669&#x202F;&#x219B;&#x202F;1676</span><span class="annotate long">line 1669 didn't jump to line 1676</span></span></p>
+    <p class="run"><span class="n"><a id="t1670" href="#t1670">1670</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.b_in"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.b_in"</span><span class="op">]</span> <span class="op">+</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1671" href="#t1671">1671</a></span><span class="t">                        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_in"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1672" href="#t1672">1672</a></span><span class="t">                        <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln2.b"</span><span class="op">]</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1673" href="#t1673">1673</a></span><span class="t">                    <span class="op">)</span><span class="op">.</span><span class="nam">sum</span><span class="op">(</span><span class="op">-</span><span class="num">2</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1674" href="#t1674">1674</a></span><span class="t">                    <span class="key">del</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln2.b"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1675" href="#t1675">1675</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1676" href="#t1676">1676</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_in"</span><span class="op">]</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1677" href="#t1677">1677</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_in"</span><span class="op">]</span> <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln2.w"</span><span class="op">]</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1678" href="#t1678">1678</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1679" href="#t1679">1679</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1680" href="#t1680">1680</a></span><span class="t">                <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">gated_mlp</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1680&#x202F;&#x219B;&#x202F;1681</span><span class="annotate long">line 1680 didn't jump to line 1681</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1681" href="#t1681">1681</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_gate"</span><span class="op">]</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1682" href="#t1682">1682</a></span><span class="t">                        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_gate"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1683" href="#t1683">1683</a></span><span class="t">                        <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln2.w"</span><span class="op">]</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1684" href="#t1684">1684</a></span><span class="t">                    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1685" href="#t1685">1685</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1686" href="#t1686">1686</a></span><span class="t">                <span class="key">del</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.ln2.w"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1687" href="#t1687">1687</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1688" href="#t1688">1688</a></span><span class="t">                <span class="key">if</span> <span class="nam">center_weights</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1688&#x202F;&#x219B;&#x202F;1696</span><span class="annotate long">line 1688 didn't jump to line 1696, because the condition on line 1688 was never false</span></span></p>
+    <p class="pln"><span class="n"><a id="t1689" href="#t1689">1689</a></span><span class="t">                    <span class="com"># Center the weights that read in from the LayerNormPre</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1690" href="#t1690">1690</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_in"</span><span class="op">]</span> <span class="op">-=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">reduce</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1691" href="#t1691">1691</a></span><span class="t">                        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_in"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1692" href="#t1692">1692</a></span><span class="t">                        <span class="str">"d_model d_mlp -> 1 d_mlp"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1693" href="#t1693">1693</a></span><span class="t">                        <span class="str">"mean"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1694" href="#t1694">1694</a></span><span class="t">                    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1695" href="#t1695">1695</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1696" href="#t1696">1696</a></span><span class="t">                <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">act_fn</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span> <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">act_fn</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"solu"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1697" href="#t1697">1697</a></span><span class="t">                    <span class="com"># Fold ln3 into activation</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1698" href="#t1698">1698</a></span><span class="t">                    <span class="key">if</span> <span class="nam">fold_biases</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1698&#x202F;&#x219B;&#x202F;1710</span><span class="annotate long">line 1698 didn't jump to line 1710</span></span></p>
+    <p class="run"><span class="n"><a id="t1699" href="#t1699">1699</a></span><span class="t">                        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.b_out"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1700" href="#t1700">1700</a></span><span class="t">                            <span class="str">f"blocks.{l}.mlp.b_out"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1701" href="#t1701">1701</a></span><span class="t">                        <span class="op">]</span> <span class="op">+</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1702" href="#t1702">1702</a></span><span class="t">                            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_out"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1703" href="#t1703">1703</a></span><span class="t">                            <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.ln.b"</span><span class="op">]</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1704" href="#t1704">1704</a></span><span class="t">                        <span class="op">)</span><span class="op">.</span><span class="nam">sum</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1705" href="#t1705">1705</a></span><span class="t">                            <span class="op">-</span><span class="num">2</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1706" href="#t1706">1706</a></span><span class="t">                        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1707" href="#t1707">1707</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1708" href="#t1708">1708</a></span><span class="t">                        <span class="key">del</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.ln.b"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1709" href="#t1709">1709</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1710" href="#t1710">1710</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_out"</span><span class="op">]</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1711" href="#t1711">1711</a></span><span class="t">                        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_out"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1712" href="#t1712">1712</a></span><span class="t">                        <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.ln.w"</span><span class="op">]</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1713" href="#t1713">1713</a></span><span class="t">                    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1714" href="#t1714">1714</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1715" href="#t1715">1715</a></span><span class="t">                    <span class="key">if</span> <span class="nam">center_weights</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1715&#x202F;&#x219B;&#x202F;1723</span><span class="annotate long">line 1715 didn't jump to line 1723, because the condition on line 1715 was never false</span></span></p>
+    <p class="pln"><span class="n"><a id="t1716" href="#t1716">1716</a></span><span class="t">                        <span class="com"># Center the weights that read in from the LayerNormPre</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1717" href="#t1717">1717</a></span><span class="t">                        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_out"</span><span class="op">]</span> <span class="op">-=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">reduce</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1718" href="#t1718">1718</a></span><span class="t">                            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_out"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1719" href="#t1719">1719</a></span><span class="t">                            <span class="str">"d_mlp d_model -> 1 d_model"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1720" href="#t1720">1720</a></span><span class="t">                            <span class="str">"mean"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1721" href="#t1721">1721</a></span><span class="t">                        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1722" href="#t1722">1722</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1723" href="#t1723">1723</a></span><span class="t">                    <span class="key">del</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.ln.w"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1724" href="#t1724">1724</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1725" href="#t1725">1725</a></span><span class="t">        <span class="com"># Fold ln_final into Unembed</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1726" href="#t1726">1726</a></span><span class="t">        <span class="key">if</span> <span class="key">not</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">final_rms</span> <span class="key">and</span> <span class="nam">fold_biases</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1727" href="#t1727">1727</a></span><span class="t">            <span class="com"># Dumb bug from my old SoLU training code, some models have RMSNorm instead of LayerNorm</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1728" href="#t1728">1728</a></span><span class="t">            <span class="com"># pre unembed.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1729" href="#t1729">1729</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"unembed.b_U"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"unembed.b_U"</span><span class="op">]</span> <span class="op">+</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1730" href="#t1730">1730</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"unembed.W_U"</span><span class="op">]</span> <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"ln_final.b"</span><span class="op">]</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1731" href="#t1731">1731</a></span><span class="t">            <span class="op">)</span><span class="op">.</span><span class="nam">sum</span><span class="op">(</span><span class="nam">dim</span><span class="op">=</span><span class="op">-</span><span class="num">2</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1732" href="#t1732">1732</a></span><span class="t">            <span class="key">del</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"ln_final.b"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1733" href="#t1733">1733</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1734" href="#t1734">1734</a></span><span class="t">        <span class="key">if</span> <span class="nam">center_weights</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1734&#x202F;&#x219B;&#x202F;1740</span><span class="annotate long">line 1734 didn't jump to line 1740, because the condition on line 1734 was never false</span></span></p>
-    <p class="pln"><span class="n"><a id="t1735" href="#t1735">1735</a></span><span class="t">            <span class="com"># Center the weights that read in from the LayerNormPre</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1736" href="#t1736">1736</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"unembed.W_U"</span><span class="op">]</span> <span class="op">-=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">reduce</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1737" href="#t1737">1737</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"unembed.W_U"</span><span class="op">]</span><span class="op">,</span> <span class="str">"d_model d_vocab -> 1 d_vocab"</span><span class="op">,</span> <span class="str">"mean"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1738" href="#t1738">1738</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1739" href="#t1739">1739</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1740" href="#t1740">1740</a></span><span class="t">        <span class="key">return</span> <span class="nam">state_dict</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1741" href="#t1741">1741</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1742" href="#t1742">1742</a></span><span class="t">    <span class="key">def</span> <span class="nam">center_writing_weights</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">state_dict</span><span class="op">:</span> <span class="nam">Dict</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">]</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1743" href="#t1743">1743</a></span><span class="t">        <span class="str">"""Center Writing Weights.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1734" href="#t1734">1734</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"unembed.W_U"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"unembed.W_U"</span><span class="op">]</span> <span class="op">*</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"ln_final.w"</span><span class="op">]</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1735" href="#t1735">1735</a></span><span class="t">        <span class="key">del</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"ln_final.w"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1736" href="#t1736">1736</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1737" href="#t1737">1737</a></span><span class="t">        <span class="key">if</span> <span class="nam">center_weights</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1737&#x202F;&#x219B;&#x202F;1743</span><span class="annotate long">line 1737 didn't jump to line 1743, because the condition on line 1737 was never false</span></span></p>
+    <p class="pln"><span class="n"><a id="t1738" href="#t1738">1738</a></span><span class="t">            <span class="com"># Center the weights that read in from the LayerNormPre</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1739" href="#t1739">1739</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"unembed.W_U"</span><span class="op">]</span> <span class="op">-=</span> <span class="nam">einops</span><span class="op">.</span><span class="nam">reduce</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1740" href="#t1740">1740</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"unembed.W_U"</span><span class="op">]</span><span class="op">,</span> <span class="str">"d_model d_vocab -> 1 d_vocab"</span><span class="op">,</span> <span class="str">"mean"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1741" href="#t1741">1741</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1742" href="#t1742">1742</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1743" href="#t1743">1743</a></span><span class="t">        <span class="key">return</span> <span class="nam">state_dict</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1744" href="#t1744">1744</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1745" href="#t1745">1745</a></span><span class="t"><span class="str">        Centers the weights of the model that write to the residual stream - W_out, W_E, W_pos and</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1746" href="#t1746">1746</a></span><span class="t"><span class="str">        W_out. This is done by subtracting the mean of the weights from the weights themselves. This</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1747" href="#t1747">1747</a></span><span class="t"><span class="str">        is done in-place. See fold_layer_norm for more details.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1748" href="#t1748">1748</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1749" href="#t1749">1749</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">[</span><span class="str">"embed.W_E"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">"embed.W_E"</span><span class="op">]</span> <span class="op">-</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">"embed.W_E"</span><span class="op">]</span><span class="op">.</span><span class="nam">mean</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1750" href="#t1750">1750</a></span><span class="t">            <span class="op">-</span><span class="num">1</span><span class="op">,</span> <span class="nam">keepdim</span><span class="op">=</span><span class="key">True</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1751" href="#t1751">1751</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1752" href="#t1752">1752</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">positional_embedding_type</span> <span class="op">!=</span> <span class="str">"rotary"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1753" href="#t1753">1753</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">"pos_embed.W_pos"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">"pos_embed.W_pos"</span><span class="op">]</span> <span class="op">-</span> <span class="nam">state_dict</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1754" href="#t1754">1754</a></span><span class="t">                <span class="str">"pos_embed.W_pos"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1755" href="#t1755">1755</a></span><span class="t">            <span class="op">]</span><span class="op">.</span><span class="nam">mean</span><span class="op">(</span><span class="op">-</span><span class="num">1</span><span class="op">,</span> <span class="nam">keepdim</span><span class="op">=</span><span class="key">True</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1756" href="#t1756">1756</a></span><span class="t">        <span class="key">for</span> <span class="nam">l</span> <span class="key">in</span> <span class="nam">range</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1757" href="#t1757">1757</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_O"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_O"</span><span class="op">]</span> <span class="op">-</span> <span class="nam">state_dict</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1758" href="#t1758">1758</a></span><span class="t">                <span class="str">f"blocks.{l}.attn.W_O"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1759" href="#t1759">1759</a></span><span class="t">            <span class="op">]</span><span class="op">.</span><span class="nam">mean</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1760" href="#t1760">1760</a></span><span class="t">                <span class="op">-</span><span class="num">1</span><span class="op">,</span> <span class="nam">keepdim</span><span class="op">=</span><span class="key">True</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1761" href="#t1761">1761</a></span><span class="t">            <span class="op">)</span>  <span class="com"># W_O is [head_index, d_model, d_head]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1762" href="#t1762">1762</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_O"</span><span class="op">]</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1763" href="#t1763">1763</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_O"</span><span class="op">]</span> <span class="op">-</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_O"</span><span class="op">]</span><span class="op">.</span><span class="nam">mean</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1764" href="#t1764">1764</a></span><span class="t">            <span class="op">)</span>  <span class="com"># b_O is [d_model]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1765" href="#t1765">1765</a></span><span class="t">            <span class="key">if</span> <span class="key">not</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">attn_only</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1766" href="#t1766">1766</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_out"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1767" href="#t1767">1767</a></span><span class="t">                    <span class="str">f"blocks.{l}.mlp.W_out"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1768" href="#t1768">1768</a></span><span class="t">                <span class="op">]</span> <span class="op">-</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_out"</span><span class="op">]</span><span class="op">.</span><span class="nam">mean</span><span class="op">(</span><span class="op">-</span><span class="num">1</span><span class="op">,</span> <span class="nam">keepdim</span><span class="op">=</span><span class="key">True</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1769" href="#t1769">1769</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.b_out"</span><span class="op">]</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1770" href="#t1770">1770</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.b_out"</span><span class="op">]</span> <span class="op">-</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.b_out"</span><span class="op">]</span><span class="op">.</span><span class="nam">mean</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1771" href="#t1771">1771</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1772" href="#t1772">1772</a></span><span class="t">        <span class="key">return</span> <span class="nam">state_dict</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1773" href="#t1773">1773</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1774" href="#t1774">1774</a></span><span class="t">    <span class="key">def</span> <span class="nam">center_unembed</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">state_dict</span><span class="op">:</span> <span class="nam">Dict</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">]</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1775" href="#t1775">1775</a></span><span class="t">        <span class="str">"""Center the unembedding weights W_U.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1745" href="#t1745">1745</a></span><span class="t">    <span class="key">def</span> <span class="nam">center_writing_weights</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">state_dict</span><span class="op">:</span> <span class="nam">Dict</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">]</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1746" href="#t1746">1746</a></span><span class="t">        <span class="str">"""Center Writing Weights.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1747" href="#t1747">1747</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1748" href="#t1748">1748</a></span><span class="t"><span class="str">        Centers the weights of the model that write to the residual stream - W_out, W_E, W_pos and</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1749" href="#t1749">1749</a></span><span class="t"><span class="str">        W_out. This is done by subtracting the mean of the weights from the weights themselves. This</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1750" href="#t1750">1750</a></span><span class="t"><span class="str">        is done in-place. See fold_layer_norm for more details.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1751" href="#t1751">1751</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1752" href="#t1752">1752</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">[</span><span class="str">"embed.W_E"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">"embed.W_E"</span><span class="op">]</span> <span class="op">-</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">"embed.W_E"</span><span class="op">]</span><span class="op">.</span><span class="nam">mean</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1753" href="#t1753">1753</a></span><span class="t">            <span class="op">-</span><span class="num">1</span><span class="op">,</span> <span class="nam">keepdim</span><span class="op">=</span><span class="key">True</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1754" href="#t1754">1754</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1755" href="#t1755">1755</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">positional_embedding_type</span> <span class="op">!=</span> <span class="str">"rotary"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1756" href="#t1756">1756</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">"pos_embed.W_pos"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">"pos_embed.W_pos"</span><span class="op">]</span> <span class="op">-</span> <span class="nam">state_dict</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1757" href="#t1757">1757</a></span><span class="t">                <span class="str">"pos_embed.W_pos"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1758" href="#t1758">1758</a></span><span class="t">            <span class="op">]</span><span class="op">.</span><span class="nam">mean</span><span class="op">(</span><span class="op">-</span><span class="num">1</span><span class="op">,</span> <span class="nam">keepdim</span><span class="op">=</span><span class="key">True</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1759" href="#t1759">1759</a></span><span class="t">        <span class="key">for</span> <span class="nam">l</span> <span class="key">in</span> <span class="nam">range</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1760" href="#t1760">1760</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_O"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_O"</span><span class="op">]</span> <span class="op">-</span> <span class="nam">state_dict</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1761" href="#t1761">1761</a></span><span class="t">                <span class="str">f"blocks.{l}.attn.W_O"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1762" href="#t1762">1762</a></span><span class="t">            <span class="op">]</span><span class="op">.</span><span class="nam">mean</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1763" href="#t1763">1763</a></span><span class="t">                <span class="op">-</span><span class="num">1</span><span class="op">,</span> <span class="nam">keepdim</span><span class="op">=</span><span class="key">True</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1764" href="#t1764">1764</a></span><span class="t">            <span class="op">)</span>  <span class="com"># W_O is [head_index, d_model, d_head]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1765" href="#t1765">1765</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_O"</span><span class="op">]</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1766" href="#t1766">1766</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_O"</span><span class="op">]</span> <span class="op">-</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_O"</span><span class="op">]</span><span class="op">.</span><span class="nam">mean</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1767" href="#t1767">1767</a></span><span class="t">            <span class="op">)</span>  <span class="com"># b_O is [d_model]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1768" href="#t1768">1768</a></span><span class="t">            <span class="key">if</span> <span class="key">not</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">attn_only</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1769" href="#t1769">1769</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_out"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1770" href="#t1770">1770</a></span><span class="t">                    <span class="str">f"blocks.{l}.mlp.W_out"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1771" href="#t1771">1771</a></span><span class="t">                <span class="op">]</span> <span class="op">-</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.W_out"</span><span class="op">]</span><span class="op">.</span><span class="nam">mean</span><span class="op">(</span><span class="op">-</span><span class="num">1</span><span class="op">,</span> <span class="nam">keepdim</span><span class="op">=</span><span class="key">True</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1772" href="#t1772">1772</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.b_out"</span><span class="op">]</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1773" href="#t1773">1773</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.b_out"</span><span class="op">]</span> <span class="op">-</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.mlp.b_out"</span><span class="op">]</span><span class="op">.</span><span class="nam">mean</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1774" href="#t1774">1774</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1775" href="#t1775">1775</a></span><span class="t">        <span class="key">return</span> <span class="nam">state_dict</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1776" href="#t1776">1776</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1777" href="#t1777">1777</a></span><span class="t"><span class="str">        This is done by subtracting the mean of the weights from the weights themselves. This is</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1778" href="#t1778">1778</a></span><span class="t"><span class="str">        done in-place. As softmax is translation invariant, this changes the logits but not the log</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1779" href="#t1779">1779</a></span><span class="t"><span class="str">        probs, and makes the model logits (slightly) more interpretable - when trying to understand</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1780" href="#t1780">1780</a></span><span class="t"><span class="str">        how components contribute to the logits, we'll be less misled by components that just add</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1781" href="#t1781">1781</a></span><span class="t"><span class="str">        something to every logit.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1782" href="#t1782">1782</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1783" href="#t1783">1783</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">[</span><span class="str">"unembed.W_U"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">"unembed.W_U"</span><span class="op">]</span> <span class="op">-</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">"unembed.W_U"</span><span class="op">]</span><span class="op">.</span><span class="nam">mean</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1784" href="#t1784">1784</a></span><span class="t">            <span class="op">-</span><span class="num">1</span><span class="op">,</span> <span class="nam">keepdim</span><span class="op">=</span><span class="key">True</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1785" href="#t1785">1785</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1786" href="#t1786">1786</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">[</span><span class="str">"unembed.b_U"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">"unembed.b_U"</span><span class="op">]</span> <span class="op">-</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">"unembed.b_U"</span><span class="op">]</span><span class="op">.</span><span class="nam">mean</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1787" href="#t1787">1787</a></span><span class="t">        <span class="key">return</span> <span class="nam">state_dict</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1788" href="#t1788">1788</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1789" href="#t1789">1789</a></span><span class="t">    <span class="key">def</span> <span class="nam">fold_value_biases</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">state_dict</span><span class="op">:</span> <span class="nam">Dict</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">]</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1790" href="#t1790">1790</a></span><span class="t">        <span class="str">"""Fold the value biases into the output bias.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1777" href="#t1777">1777</a></span><span class="t">    <span class="key">def</span> <span class="nam">center_unembed</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">state_dict</span><span class="op">:</span> <span class="nam">Dict</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">]</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1778" href="#t1778">1778</a></span><span class="t">        <span class="str">"""Center the unembedding weights W_U.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1779" href="#t1779">1779</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1780" href="#t1780">1780</a></span><span class="t"><span class="str">        This is done by subtracting the mean of the weights from the weights themselves. This is</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1781" href="#t1781">1781</a></span><span class="t"><span class="str">        done in-place. As softmax is translation invariant, this changes the logits but not the log</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1782" href="#t1782">1782</a></span><span class="t"><span class="str">        probs, and makes the model logits (slightly) more interpretable - when trying to understand</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1783" href="#t1783">1783</a></span><span class="t"><span class="str">        how components contribute to the logits, we'll be less misled by components that just add</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1784" href="#t1784">1784</a></span><span class="t"><span class="str">        something to every logit.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1785" href="#t1785">1785</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1786" href="#t1786">1786</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">[</span><span class="str">"unembed.W_U"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">"unembed.W_U"</span><span class="op">]</span> <span class="op">-</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">"unembed.W_U"</span><span class="op">]</span><span class="op">.</span><span class="nam">mean</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1787" href="#t1787">1787</a></span><span class="t">            <span class="op">-</span><span class="num">1</span><span class="op">,</span> <span class="nam">keepdim</span><span class="op">=</span><span class="key">True</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1788" href="#t1788">1788</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1789" href="#t1789">1789</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">[</span><span class="str">"unembed.b_U"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">"unembed.b_U"</span><span class="op">]</span> <span class="op">-</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">"unembed.b_U"</span><span class="op">]</span><span class="op">.</span><span class="nam">mean</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1790" href="#t1790">1790</a></span><span class="t">        <span class="key">return</span> <span class="nam">state_dict</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1791" href="#t1791">1791</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1792" href="#t1792">1792</a></span><span class="t"><span class="str">        Because attention patterns add up to 1, the value biases always have a constant effect on a</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1793" href="#t1793">1793</a></span><span class="t"><span class="str">        head's output. Further, as the outputs of each head in a layer add together, each head's</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1794" href="#t1794">1794</a></span><span class="t"><span class="str">        value bias has a constant effect on the *layer's* output, which can make it harder to</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1795" href="#t1795">1795</a></span><span class="t"><span class="str">        interpret the effect of any given head, and it doesn't matter which head a bias is</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1796" href="#t1796">1796</a></span><span class="t"><span class="str">        associated with. We can factor this all into a single output bias to the layer, and make it</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1797" href="#t1797">1797</a></span><span class="t"><span class="str">        easier to interpret the head's output. Formally, we take b_O_new = b_O_original +</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1798" href="#t1798">1798</a></span><span class="t"><span class="str">        sum_head(b_V_head @ W_O_head).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1799" href="#t1799">1799</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1800" href="#t1800">1800</a></span><span class="t">        <span class="key">for</span> <span class="nam">layer</span> <span class="key">in</span> <span class="nam">range</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1801" href="#t1801">1801</a></span><span class="t">            <span class="com"># shape [head_index, d_head]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1802" href="#t1802">1802</a></span><span class="t">            <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_key_value_heads</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1802&#x202F;&#x219B;&#x202F;1805</span><span class="annotate long">line 1802 didn't jump to line 1805, because the condition on line 1802 was never false</span></span></p>
-    <p class="run"><span class="n"><a id="t1803" href="#t1803">1803</a></span><span class="t">                <span class="nam">b_V</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{layer}.attn.b_V"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1804" href="#t1804">1804</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1805" href="#t1805">1805</a></span><span class="t">                <span class="nam">b_V</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{layer}.attn._b_V"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1806" href="#t1806">1806</a></span><span class="t">                <span class="nam">b_V</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">repeat_interleave</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1807" href="#t1807">1807</a></span><span class="t">                    <span class="nam">b_V</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">,</span> <span class="nam">repeats</span><span class="op">=</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_heads</span> <span class="op">//</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_key_value_heads</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1808" href="#t1808">1808</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1809" href="#t1809">1809</a></span><span class="t">            <span class="com"># [head_index, d_head, d_model]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1810" href="#t1810">1810</a></span><span class="t">            <span class="nam">W_O</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{layer}.attn.W_O"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1811" href="#t1811">1811</a></span><span class="t">            <span class="com"># [d_model]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1812" href="#t1812">1812</a></span><span class="t">            <span class="nam">b_O_original</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{layer}.attn.b_O"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1813" href="#t1813">1813</a></span><span class="t">            <span class="nam">folded_b_O</span> <span class="op">=</span> <span class="nam">b_O_original</span> <span class="op">+</span> <span class="op">(</span><span class="nam">b_V</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span> <span class="op">*</span> <span class="nam">W_O</span><span class="op">)</span><span class="op">.</span><span class="nam">sum</span><span class="op">(</span><span class="op">[</span><span class="num">0</span><span class="op">,</span> <span class="num">1</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1814" href="#t1814">1814</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1815" href="#t1815">1815</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{layer}.attn.b_O"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">folded_b_O</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1816" href="#t1816">1816</a></span><span class="t">            <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_key_value_heads</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1816&#x202F;&#x219B;&#x202F;1819</span><span class="annotate long">line 1816 didn't jump to line 1819, because the condition on line 1816 was never false</span></span></p>
-    <p class="run"><span class="n"><a id="t1817" href="#t1817">1817</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{layer}.attn.b_V"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">zeros_like</span><span class="op">(</span><span class="nam">b_V</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1818" href="#t1818">1818</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1819" href="#t1819">1819</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{layer}.attn._b_V"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">zeros_like</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1820" href="#t1820">1820</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{layer}.attn._b_V"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1821" href="#t1821">1821</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1822" href="#t1822">1822</a></span><span class="t">        <span class="key">return</span> <span class="nam">state_dict</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1823" href="#t1823">1823</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1824" href="#t1824">1824</a></span><span class="t">    <span class="key">def</span> <span class="nam">refactor_factored_attn_matrices</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">state_dict</span><span class="op">:</span> <span class="nam">Dict</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">]</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1825" href="#t1825">1825</a></span><span class="t">        <span class="str">"""Experimental method for managing queries, keys and values.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1792" href="#t1792">1792</a></span><span class="t">    <span class="key">def</span> <span class="nam">fold_value_biases</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">state_dict</span><span class="op">:</span> <span class="nam">Dict</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">]</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1793" href="#t1793">1793</a></span><span class="t">        <span class="str">"""Fold the value biases into the output bias.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1794" href="#t1794">1794</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1795" href="#t1795">1795</a></span><span class="t"><span class="str">        Because attention patterns add up to 1, the value biases always have a constant effect on a</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1796" href="#t1796">1796</a></span><span class="t"><span class="str">        head's output. Further, as the outputs of each head in a layer add together, each head's</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1797" href="#t1797">1797</a></span><span class="t"><span class="str">        value bias has a constant effect on the *layer's* output, which can make it harder to</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1798" href="#t1798">1798</a></span><span class="t"><span class="str">        interpret the effect of any given head, and it doesn't matter which head a bias is</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1799" href="#t1799">1799</a></span><span class="t"><span class="str">        associated with. We can factor this all into a single output bias to the layer, and make it</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1800" href="#t1800">1800</a></span><span class="t"><span class="str">        easier to interpret the head's output. Formally, we take b_O_new = b_O_original +</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1801" href="#t1801">1801</a></span><span class="t"><span class="str">        sum_head(b_V_head @ W_O_head).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1802" href="#t1802">1802</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1803" href="#t1803">1803</a></span><span class="t">        <span class="key">for</span> <span class="nam">layer</span> <span class="key">in</span> <span class="nam">range</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1804" href="#t1804">1804</a></span><span class="t">            <span class="com"># shape [head_index, d_head]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1805" href="#t1805">1805</a></span><span class="t">            <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_key_value_heads</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1805&#x202F;&#x219B;&#x202F;1808</span><span class="annotate long">line 1805 didn't jump to line 1808, because the condition on line 1805 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t1806" href="#t1806">1806</a></span><span class="t">                <span class="nam">b_V</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{layer}.attn.b_V"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1807" href="#t1807">1807</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1808" href="#t1808">1808</a></span><span class="t">                <span class="nam">b_V</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{layer}.attn._b_V"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1809" href="#t1809">1809</a></span><span class="t">                <span class="nam">b_V</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">repeat_interleave</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1810" href="#t1810">1810</a></span><span class="t">                    <span class="nam">b_V</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">,</span> <span class="nam">repeats</span><span class="op">=</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_heads</span> <span class="op">//</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_key_value_heads</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1811" href="#t1811">1811</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1812" href="#t1812">1812</a></span><span class="t">            <span class="com"># [head_index, d_head, d_model]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1813" href="#t1813">1813</a></span><span class="t">            <span class="nam">W_O</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{layer}.attn.W_O"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1814" href="#t1814">1814</a></span><span class="t">            <span class="com"># [d_model]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1815" href="#t1815">1815</a></span><span class="t">            <span class="nam">b_O_original</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{layer}.attn.b_O"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1816" href="#t1816">1816</a></span><span class="t">            <span class="nam">folded_b_O</span> <span class="op">=</span> <span class="nam">b_O_original</span> <span class="op">+</span> <span class="op">(</span><span class="nam">b_V</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span> <span class="op">*</span> <span class="nam">W_O</span><span class="op">)</span><span class="op">.</span><span class="nam">sum</span><span class="op">(</span><span class="op">[</span><span class="num">0</span><span class="op">,</span> <span class="num">1</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1817" href="#t1817">1817</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1818" href="#t1818">1818</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{layer}.attn.b_O"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">folded_b_O</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1819" href="#t1819">1819</a></span><span class="t">            <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_key_value_heads</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1819&#x202F;&#x219B;&#x202F;1822</span><span class="annotate long">line 1819 didn't jump to line 1822, because the condition on line 1819 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t1820" href="#t1820">1820</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{layer}.attn.b_V"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">zeros_like</span><span class="op">(</span><span class="nam">b_V</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1821" href="#t1821">1821</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1822" href="#t1822">1822</a></span><span class="t">                <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{layer}.attn._b_V"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">zeros_like</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1823" href="#t1823">1823</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{layer}.attn._b_V"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1824" href="#t1824">1824</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1825" href="#t1825">1825</a></span><span class="t">        <span class="key">return</span> <span class="nam">state_dict</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1826" href="#t1826">1826</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1827" href="#t1827">1827</a></span><span class="t"><span class="str">        As argued in [A Mathematical Framework for Transformer</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1828" href="#t1828">1828</a></span><span class="t"><span class="str">        Circuits](https://transformer-circuits.pub/2021/framework/index.html), queries, keys and</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1829" href="#t1829">1829</a></span><span class="t"><span class="str">        values are somewhat arbitrary intermediate terms when computing with the low rank factored</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1830" href="#t1830">1830</a></span><span class="t"><span class="str">        matrices W_QK = W_Q @ W_K.T and W_OV = W_V @ W_O, and these matrices are the only thing</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1831" href="#t1831">1831</a></span><span class="t"><span class="str">        determining head behaviour. But there are many ways to find a low rank factorization to a</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1832" href="#t1832">1832</a></span><span class="t"><span class="str">        given matrix, and hopefully some of these are more interpretable than others! This method is</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1833" href="#t1833">1833</a></span><span class="t"><span class="str">        one attempt, which makes all of the matrices have orthogonal rows or columns, W_O into a</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1834" href="#t1834">1834</a></span><span class="t"><span class="str">        rotation and W_Q and W_K having the nth column in each having the same norm. The formula is</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1835" href="#t1835">1835</a></span><span class="t"><span class="str">        $W_V = U @ S,W_O=Vh.T,W_Q=U@S.sqrt(),W_K=Vh@S.sqrt()$.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1836" href="#t1836">1836</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1837" href="#t1837">1837</a></span><span class="t"><span class="str">        More details:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1838" href="#t1838">1838</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1839" href="#t1839">1839</a></span><span class="t"><span class="str">        If W_OV = U @ S @ Vh.T in its singular value decomposition, (where S is in R^d_head not</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1840" href="#t1840">1840</a></span><span class="t"><span class="str">        R^d_model, as W_OV is low rank), W_OV = (U @ S) @ (Vh.T) is an equivalent low rank</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1841" href="#t1841">1841</a></span><span class="t"><span class="str">        factorisation, where rows/columns of each matrix are orthogonal! So setting $W_V=US$ and</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1842" href="#t1842">1842</a></span><span class="t"><span class="str">        $W_O=Vh.T$ works just as well. I *think* this is a more interpretable setup, because now</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1843" href="#t1843">1843</a></span><span class="t"><span class="str">        $W_O$ is just a rotation, and doesn't change the norm, so $z$ has the same norm as the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1844" href="#t1844">1844</a></span><span class="t"><span class="str">        result of the head.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1845" href="#t1845">1845</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1846" href="#t1846">1846</a></span><span class="t"><span class="str">        For $W_QK = W_Q @ W_K.T$ we use the refactor $W_Q = U @ S.sqrt()$ and $W_K = Vh @ S.sqrt()$,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1847" href="#t1847">1847</a></span><span class="t"><span class="str">        which is also equivalent ($S==S.sqrt() @ S.sqrt()$ as $S$ is diagonal). Here we keep the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1848" href="#t1848">1848</a></span><span class="t"><span class="str">        matrices as having the same norm, since there's not an obvious asymmetry between the keys</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1849" href="#t1849">1849</a></span><span class="t"><span class="str">        and queries.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1850" href="#t1850">1850</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1851" href="#t1851">1851</a></span><span class="t"><span class="str">        Biases are more fiddly to deal with. For OV it's pretty easy - we just need (x @ W_V + b_V)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1852" href="#t1852">1852</a></span><span class="t"><span class="str">        @ W_O + b_O to be preserved, so we can set b_V' = 0. and b_O' = b_V @ W_O + b_O (note that</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1853" href="#t1853">1853</a></span><span class="t"><span class="str">        b_V in R^{head_index x d_head} while b_O in R^{d_model}, so we need to sum b_V @ W_O along</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1854" href="#t1854">1854</a></span><span class="t"><span class="str">        the head_index dimension too).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1855" href="#t1855">1855</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1856" href="#t1856">1856</a></span><span class="t"><span class="str">        For QK it's messy - we need to preserve the bilinear form of (x @ W_Q + b_Q) * (y @ W_K +</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1857" href="#t1857">1857</a></span><span class="t"><span class="str">        b_K), which is fairly messy. To deal with the biases, we concatenate them to W_Q and W_K to</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1858" href="#t1858">1858</a></span><span class="t"><span class="str">        simulate a d_model+1 dimensional input (whose final coordinate is always 1), do the SVD</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1859" href="#t1859">1859</a></span><span class="t"><span class="str">        factorization on this effective matrix, then separate out into final weights and biases.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1860" href="#t1860">1860</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1861" href="#t1861">1861</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1862" href="#t1862">1862</a></span><span class="t">        <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1863" href="#t1863">1863</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">positional_embedding_type</span> <span class="op">!=</span> <span class="str">"rotary"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1864" href="#t1864">1864</a></span><span class="t">        <span class="op">)</span><span class="op">,</span> <span class="str">"You can't refactor the QK circuit when using rotary embeddings (as the QK matrix depends on the position of the query and key)"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1865" href="#t1865">1865</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1866" href="#t1866">1866</a></span><span class="t">        <span class="key">for</span> <span class="nam">l</span> <span class="key">in</span> <span class="nam">range</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1867" href="#t1867">1867</a></span><span class="t">            <span class="com"># W_QK = W_Q @ W_K.T</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1868" href="#t1868">1868</a></span><span class="t">            <span class="com"># Concatenate biases to make a d_model+1 input dimension</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1869" href="#t1869">1869</a></span><span class="t">            <span class="nam">W_Q_eff</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">cat</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1870" href="#t1870">1870</a></span><span class="t">                <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1871" href="#t1871">1871</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_Q"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1872" href="#t1872">1872</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_Q"</span><span class="op">]</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">,</span> <span class="op">:</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1873" href="#t1873">1873</a></span><span class="t">                <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1874" href="#t1874">1874</a></span><span class="t">                <span class="nam">dim</span><span class="op">=</span><span class="num">1</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1875" href="#t1875">1875</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1876" href="#t1876">1876</a></span><span class="t">            <span class="nam">W_K_eff</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">cat</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1877" href="#t1877">1877</a></span><span class="t">                <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1878" href="#t1878">1878</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_K"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1879" href="#t1879">1879</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_K"</span><span class="op">]</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">,</span> <span class="op">:</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1880" href="#t1880">1880</a></span><span class="t">                <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1881" href="#t1881">1881</a></span><span class="t">                <span class="nam">dim</span><span class="op">=</span><span class="num">1</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1882" href="#t1882">1882</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1883" href="#t1883">1883</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1884" href="#t1884">1884</a></span><span class="t">            <span class="nam">W_Q_eff_even</span><span class="op">,</span> <span class="nam">W_K_eff_even_T</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1885" href="#t1885">1885</a></span><span class="t">                <span class="nam">FactoredMatrix</span><span class="op">(</span><span class="nam">W_Q_eff</span><span class="op">,</span> <span class="nam">W_K_eff</span><span class="op">.</span><span class="nam">transpose</span><span class="op">(</span><span class="op">-</span><span class="num">1</span><span class="op">,</span> <span class="op">-</span><span class="num">2</span><span class="op">)</span><span class="op">)</span><span class="op">.</span><span class="nam">make_even</span><span class="op">(</span><span class="op">)</span><span class="op">.</span><span class="nam">pair</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1886" href="#t1886">1886</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1887" href="#t1887">1887</a></span><span class="t">            <span class="nam">W_K_eff_even</span> <span class="op">=</span> <span class="nam">W_K_eff_even_T</span><span class="op">.</span><span class="nam">transpose</span><span class="op">(</span><span class="op">-</span><span class="num">1</span><span class="op">,</span> <span class="op">-</span><span class="num">2</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1888" href="#t1888">1888</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1889" href="#t1889">1889</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_Q"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">W_Q_eff_even</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="op">:</span><span class="op">-</span><span class="num">1</span><span class="op">,</span> <span class="op">:</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1890" href="#t1890">1890</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_Q"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">W_Q_eff_even</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="op">-</span><span class="num">1</span><span class="op">,</span> <span class="op">:</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1891" href="#t1891">1891</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_K"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">W_K_eff_even</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="op">:</span><span class="op">-</span><span class="num">1</span><span class="op">,</span> <span class="op">:</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1892" href="#t1892">1892</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_K"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">W_K_eff_even</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="op">-</span><span class="num">1</span><span class="op">,</span> <span class="op">:</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1893" href="#t1893">1893</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1894" href="#t1894">1894</a></span><span class="t">            <span class="com"># W_OV = W_V @ W_O</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1895" href="#t1895">1895</a></span><span class="t">            <span class="nam">W_V</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_V"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1896" href="#t1896">1896</a></span><span class="t">            <span class="nam">W_O</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_O"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1897" href="#t1897">1897</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1898" href="#t1898">1898</a></span><span class="t">            <span class="com"># Factors the bias to be consistent.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1899" href="#t1899">1899</a></span><span class="t">            <span class="nam">b_V</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_V"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1900" href="#t1900">1900</a></span><span class="t">            <span class="nam">b_O</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_O"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1901" href="#t1901">1901</a></span><span class="t">            <span class="nam">effective_bias</span> <span class="op">=</span> <span class="nam">b_O</span> <span class="op">+</span> <span class="nam">einsum</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1902" href="#t1902">1902</a></span><span class="t">                <span class="str">"head_index d_head, head_index d_head d_model -> d_model"</span><span class="op">,</span> <span class="nam">b_V</span><span class="op">,</span> <span class="nam">W_O</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1903" href="#t1903">1903</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1904" href="#t1904">1904</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_V"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">zeros_like</span><span class="op">(</span><span class="nam">b_V</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1905" href="#t1905">1905</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_O"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">effective_bias</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1906" href="#t1906">1906</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1907" href="#t1907">1907</a></span><span class="t">            <span class="com"># Helper class to efficiently deal with low rank factored matrices.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1908" href="#t1908">1908</a></span><span class="t">            <span class="nam">W_OV</span> <span class="op">=</span> <span class="nam">FactoredMatrix</span><span class="op">(</span><span class="nam">W_V</span><span class="op">,</span> <span class="nam">W_O</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1909" href="#t1909">1909</a></span><span class="t">            <span class="nam">U</span><span class="op">,</span> <span class="nam">S</span><span class="op">,</span> <span class="nam">Vh</span> <span class="op">=</span> <span class="nam">W_OV</span><span class="op">.</span><span class="nam">svd</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1910" href="#t1910">1910</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_V"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">U</span> <span class="op">@</span> <span class="nam">S</span><span class="op">.</span><span class="nam">diag_embed</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1911" href="#t1911">1911</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_O"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">transpose</span><span class="op">(</span><span class="nam">Vh</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1912" href="#t1912">1912</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1913" href="#t1913">1913</a></span><span class="t">        <span class="key">return</span> <span class="nam">state_dict</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1914" href="#t1914">1914</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1915" href="#t1915">1915</a></span><span class="t">    <span class="key">def</span> <span class="nam">set_use_attn_result</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">use_attn_result</span><span class="op">:</span> <span class="nam">bool</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1916" href="#t1916">1916</a></span><span class="t">        <span class="str">"""Toggle whether to explicitly calculate and expose the result for each attention head.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1827" href="#t1827">1827</a></span><span class="t">    <span class="key">def</span> <span class="nam">refactor_factored_attn_matrices</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">state_dict</span><span class="op">:</span> <span class="nam">Dict</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">]</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1828" href="#t1828">1828</a></span><span class="t">        <span class="str">"""Experimental method for managing queries, keys and values.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1829" href="#t1829">1829</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1830" href="#t1830">1830</a></span><span class="t"><span class="str">        As argued in [A Mathematical Framework for Transformer</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1831" href="#t1831">1831</a></span><span class="t"><span class="str">        Circuits](https://transformer-circuits.pub/2021/framework/index.html), queries, keys and</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1832" href="#t1832">1832</a></span><span class="t"><span class="str">        values are somewhat arbitrary intermediate terms when computing with the low rank factored</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1833" href="#t1833">1833</a></span><span class="t"><span class="str">        matrices W_QK = W_Q @ W_K.T and W_OV = W_V @ W_O, and these matrices are the only thing</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1834" href="#t1834">1834</a></span><span class="t"><span class="str">        determining head behaviour. But there are many ways to find a low rank factorization to a</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1835" href="#t1835">1835</a></span><span class="t"><span class="str">        given matrix, and hopefully some of these are more interpretable than others! This method is</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1836" href="#t1836">1836</a></span><span class="t"><span class="str">        one attempt, which makes all of the matrices have orthogonal rows or columns, W_O into a</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1837" href="#t1837">1837</a></span><span class="t"><span class="str">        rotation and W_Q and W_K having the nth column in each having the same norm. The formula is</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1838" href="#t1838">1838</a></span><span class="t"><span class="str">        $W_V = U @ S,W_O=Vh.T,W_Q=U@S.sqrt(),W_K=Vh@S.sqrt()$.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1839" href="#t1839">1839</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1840" href="#t1840">1840</a></span><span class="t"><span class="str">        More details:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1841" href="#t1841">1841</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1842" href="#t1842">1842</a></span><span class="t"><span class="str">        If W_OV = U @ S @ Vh.T in its singular value decomposition, (where S is in R^d_head not</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1843" href="#t1843">1843</a></span><span class="t"><span class="str">        R^d_model, as W_OV is low rank), W_OV = (U @ S) @ (Vh.T) is an equivalent low rank</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1844" href="#t1844">1844</a></span><span class="t"><span class="str">        factorisation, where rows/columns of each matrix are orthogonal! So setting $W_V=US$ and</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1845" href="#t1845">1845</a></span><span class="t"><span class="str">        $W_O=Vh.T$ works just as well. I *think* this is a more interpretable setup, because now</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1846" href="#t1846">1846</a></span><span class="t"><span class="str">        $W_O$ is just a rotation, and doesn't change the norm, so $z$ has the same norm as the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1847" href="#t1847">1847</a></span><span class="t"><span class="str">        result of the head.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1848" href="#t1848">1848</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1849" href="#t1849">1849</a></span><span class="t"><span class="str">        For $W_QK = W_Q @ W_K.T$ we use the refactor $W_Q = U @ S.sqrt()$ and $W_K = Vh @ S.sqrt()$,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1850" href="#t1850">1850</a></span><span class="t"><span class="str">        which is also equivalent ($S==S.sqrt() @ S.sqrt()$ as $S$ is diagonal). Here we keep the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1851" href="#t1851">1851</a></span><span class="t"><span class="str">        matrices as having the same norm, since there's not an obvious asymmetry between the keys</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1852" href="#t1852">1852</a></span><span class="t"><span class="str">        and queries.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1853" href="#t1853">1853</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1854" href="#t1854">1854</a></span><span class="t"><span class="str">        Biases are more fiddly to deal with. For OV it's pretty easy - we just need (x @ W_V + b_V)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1855" href="#t1855">1855</a></span><span class="t"><span class="str">        @ W_O + b_O to be preserved, so we can set b_V' = 0. and b_O' = b_V @ W_O + b_O (note that</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1856" href="#t1856">1856</a></span><span class="t"><span class="str">        b_V in R^{head_index x d_head} while b_O in R^{d_model}, so we need to sum b_V @ W_O along</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1857" href="#t1857">1857</a></span><span class="t"><span class="str">        the head_index dimension too).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1858" href="#t1858">1858</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1859" href="#t1859">1859</a></span><span class="t"><span class="str">        For QK it's messy - we need to preserve the bilinear form of (x @ W_Q + b_Q) * (y @ W_K +</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1860" href="#t1860">1860</a></span><span class="t"><span class="str">        b_K), which is fairly messy. To deal with the biases, we concatenate them to W_Q and W_K to</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1861" href="#t1861">1861</a></span><span class="t"><span class="str">        simulate a d_model+1 dimensional input (whose final coordinate is always 1), do the SVD</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1862" href="#t1862">1862</a></span><span class="t"><span class="str">        factorization on this effective matrix, then separate out into final weights and biases.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1863" href="#t1863">1863</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1864" href="#t1864">1864</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1865" href="#t1865">1865</a></span><span class="t">        <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1866" href="#t1866">1866</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">positional_embedding_type</span> <span class="op">!=</span> <span class="str">"rotary"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1867" href="#t1867">1867</a></span><span class="t">        <span class="op">)</span><span class="op">,</span> <span class="str">"You can't refactor the QK circuit when using rotary embeddings (as the QK matrix depends on the position of the query and key)"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1868" href="#t1868">1868</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1869" href="#t1869">1869</a></span><span class="t">        <span class="key">for</span> <span class="nam">l</span> <span class="key">in</span> <span class="nam">range</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1870" href="#t1870">1870</a></span><span class="t">            <span class="com"># W_QK = W_Q @ W_K.T</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1871" href="#t1871">1871</a></span><span class="t">            <span class="com"># Concatenate biases to make a d_model+1 input dimension</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1872" href="#t1872">1872</a></span><span class="t">            <span class="nam">W_Q_eff</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">cat</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1873" href="#t1873">1873</a></span><span class="t">                <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1874" href="#t1874">1874</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_Q"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1875" href="#t1875">1875</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_Q"</span><span class="op">]</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">,</span> <span class="op">:</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1876" href="#t1876">1876</a></span><span class="t">                <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1877" href="#t1877">1877</a></span><span class="t">                <span class="nam">dim</span><span class="op">=</span><span class="num">1</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1878" href="#t1878">1878</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1879" href="#t1879">1879</a></span><span class="t">            <span class="nam">W_K_eff</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">cat</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1880" href="#t1880">1880</a></span><span class="t">                <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1881" href="#t1881">1881</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_K"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1882" href="#t1882">1882</a></span><span class="t">                    <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_K"</span><span class="op">]</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">,</span> <span class="op">:</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1883" href="#t1883">1883</a></span><span class="t">                <span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1884" href="#t1884">1884</a></span><span class="t">                <span class="nam">dim</span><span class="op">=</span><span class="num">1</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1885" href="#t1885">1885</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1886" href="#t1886">1886</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1887" href="#t1887">1887</a></span><span class="t">            <span class="nam">W_Q_eff_even</span><span class="op">,</span> <span class="nam">W_K_eff_even_T</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1888" href="#t1888">1888</a></span><span class="t">                <span class="nam">FactoredMatrix</span><span class="op">(</span><span class="nam">W_Q_eff</span><span class="op">,</span> <span class="nam">W_K_eff</span><span class="op">.</span><span class="nam">transpose</span><span class="op">(</span><span class="op">-</span><span class="num">1</span><span class="op">,</span> <span class="op">-</span><span class="num">2</span><span class="op">)</span><span class="op">)</span><span class="op">.</span><span class="nam">make_even</span><span class="op">(</span><span class="op">)</span><span class="op">.</span><span class="nam">pair</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1889" href="#t1889">1889</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1890" href="#t1890">1890</a></span><span class="t">            <span class="nam">W_K_eff_even</span> <span class="op">=</span> <span class="nam">W_K_eff_even_T</span><span class="op">.</span><span class="nam">transpose</span><span class="op">(</span><span class="op">-</span><span class="num">1</span><span class="op">,</span> <span class="op">-</span><span class="num">2</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1891" href="#t1891">1891</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1892" href="#t1892">1892</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_Q"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">W_Q_eff_even</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="op">:</span><span class="op">-</span><span class="num">1</span><span class="op">,</span> <span class="op">:</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1893" href="#t1893">1893</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_Q"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">W_Q_eff_even</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="op">-</span><span class="num">1</span><span class="op">,</span> <span class="op">:</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1894" href="#t1894">1894</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_K"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">W_K_eff_even</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="op">:</span><span class="op">-</span><span class="num">1</span><span class="op">,</span> <span class="op">:</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1895" href="#t1895">1895</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_K"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">W_K_eff_even</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="op">-</span><span class="num">1</span><span class="op">,</span> <span class="op">:</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1896" href="#t1896">1896</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1897" href="#t1897">1897</a></span><span class="t">            <span class="com"># W_OV = W_V @ W_O</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1898" href="#t1898">1898</a></span><span class="t">            <span class="nam">W_V</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_V"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1899" href="#t1899">1899</a></span><span class="t">            <span class="nam">W_O</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_O"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1900" href="#t1900">1900</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1901" href="#t1901">1901</a></span><span class="t">            <span class="com"># Factors the bias to be consistent.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1902" href="#t1902">1902</a></span><span class="t">            <span class="nam">b_V</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_V"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1903" href="#t1903">1903</a></span><span class="t">            <span class="nam">b_O</span> <span class="op">=</span> <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_O"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1904" href="#t1904">1904</a></span><span class="t">            <span class="nam">effective_bias</span> <span class="op">=</span> <span class="nam">b_O</span> <span class="op">+</span> <span class="nam">einsum</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1905" href="#t1905">1905</a></span><span class="t">                <span class="str">"head_index d_head, head_index d_head d_model -> d_model"</span><span class="op">,</span> <span class="nam">b_V</span><span class="op">,</span> <span class="nam">W_O</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1906" href="#t1906">1906</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1907" href="#t1907">1907</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_V"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">zeros_like</span><span class="op">(</span><span class="nam">b_V</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1908" href="#t1908">1908</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.b_O"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">effective_bias</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1909" href="#t1909">1909</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1910" href="#t1910">1910</a></span><span class="t">            <span class="com"># Helper class to efficiently deal with low rank factored matrices.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1911" href="#t1911">1911</a></span><span class="t">            <span class="nam">W_OV</span> <span class="op">=</span> <span class="nam">FactoredMatrix</span><span class="op">(</span><span class="nam">W_V</span><span class="op">,</span> <span class="nam">W_O</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1912" href="#t1912">1912</a></span><span class="t">            <span class="nam">U</span><span class="op">,</span> <span class="nam">S</span><span class="op">,</span> <span class="nam">Vh</span> <span class="op">=</span> <span class="nam">W_OV</span><span class="op">.</span><span class="nam">svd</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1913" href="#t1913">1913</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_V"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">U</span> <span class="op">@</span> <span class="nam">S</span><span class="op">.</span><span class="nam">diag_embed</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1914" href="#t1914">1914</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">[</span><span class="str">f"blocks.{l}.attn.W_O"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">transpose</span><span class="op">(</span><span class="nam">Vh</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1915" href="#t1915">1915</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1916" href="#t1916">1916</a></span><span class="t">        <span class="key">return</span> <span class="nam">state_dict</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1917" href="#t1917">1917</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1918" href="#t1918">1918</a></span><span class="t"><span class="str">        Useful for interpretability but can easily burn through GPU memory.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1919" href="#t1919">1919</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1920" href="#t1920">1920</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">use_attn_result</span> <span class="op">=</span> <span class="nam">use_attn_result</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1921" href="#t1921">1921</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1922" href="#t1922">1922</a></span><span class="t">    <span class="key">def</span> <span class="nam">set_use_split_qkv_input</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">use_split_qkv_input</span><span class="op">:</span> <span class="nam">bool</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1923" href="#t1923">1923</a></span><span class="t">        <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1924" href="#t1924">1924</a></span><span class="t"><span class="str">        Toggles whether to allow editing of inputs to each attention head.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1925" href="#t1925">1925</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1926" href="#t1926">1926</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">use_split_qkv_input</span> <span class="op">=</span> <span class="nam">use_split_qkv_input</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1927" href="#t1927">1927</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1928" href="#t1928">1928</a></span><span class="t">    <span class="key">def</span> <span class="nam">set_use_hook_mlp_in</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">use_hook_mlp_in</span><span class="op">:</span> <span class="nam">bool</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1929" href="#t1929">1929</a></span><span class="t">        <span class="str">"""Toggles whether to allow storing and editing inputs to each MLP layer."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1918" href="#t1918">1918</a></span><span class="t">    <span class="key">def</span> <span class="nam">set_use_attn_result</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">use_attn_result</span><span class="op">:</span> <span class="nam">bool</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1919" href="#t1919">1919</a></span><span class="t">        <span class="str">"""Toggle whether to explicitly calculate and expose the result for each attention head.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1920" href="#t1920">1920</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1921" href="#t1921">1921</a></span><span class="t"><span class="str">        Useful for interpretability but can easily burn through GPU memory.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1922" href="#t1922">1922</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1923" href="#t1923">1923</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">use_attn_result</span> <span class="op">=</span> <span class="nam">use_attn_result</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1924" href="#t1924">1924</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1925" href="#t1925">1925</a></span><span class="t">    <span class="key">def</span> <span class="nam">set_use_split_qkv_input</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">use_split_qkv_input</span><span class="op">:</span> <span class="nam">bool</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1926" href="#t1926">1926</a></span><span class="t">        <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1927" href="#t1927">1927</a></span><span class="t"><span class="str">        Toggles whether to allow editing of inputs to each attention head.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1928" href="#t1928">1928</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1929" href="#t1929">1929</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">use_split_qkv_input</span> <span class="op">=</span> <span class="nam">use_split_qkv_input</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1930" href="#t1930">1930</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1931" href="#t1931">1931</a></span><span class="t">        <span class="key">assert</span> <span class="key">not</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">attn_only</span><span class="op">,</span> <span class="str">"Can't use hook_mlp_in with attn_only model"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1932" href="#t1932">1932</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">use_hook_mlp_in</span> <span class="op">=</span> <span class="nam">use_hook_mlp_in</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1931" href="#t1931">1931</a></span><span class="t">    <span class="key">def</span> <span class="nam">set_use_hook_mlp_in</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">use_hook_mlp_in</span><span class="op">:</span> <span class="nam">bool</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1932" href="#t1932">1932</a></span><span class="t">        <span class="str">"""Toggles whether to allow storing and editing inputs to each MLP layer."""</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1933" href="#t1933">1933</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1934" href="#t1934">1934</a></span><span class="t">    <span class="key">def</span> <span class="nam">set_use_attn_in</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">use_attn_in</span><span class="op">:</span> <span class="nam">bool</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1935" href="#t1935">1935</a></span><span class="t">        <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1936" href="#t1936">1936</a></span><span class="t"><span class="str">        Toggles whether to allow editing of inputs to each attention head.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1937" href="#t1937">1937</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1938" href="#t1938">1938</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">use_attn_in</span> <span class="op">=</span> <span class="nam">use_attn_in</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1939" href="#t1939">1939</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1940" href="#t1940">1940</a></span><span class="t">    <span class="key">def</span> <span class="nam">process_weights_</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1941" href="#t1941">1941</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1942" href="#t1942">1942</a></span><span class="t">        <span class="nam">fold_ln</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1943" href="#t1943">1943</a></span><span class="t">        <span class="nam">center_writing_weights</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1944" href="#t1944">1944</a></span><span class="t">        <span class="nam">center_unembed</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1945" href="#t1945">1945</a></span><span class="t">        <span class="nam">refactor_factored_attn_matrices</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1946" href="#t1946">1946</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1947" href="#t1947">1947</a></span><span class="t">        <span class="str">"""Wrapper around `load_and_process_state_dict`.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1948" href="#t1948">1948</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1949" href="#t1949">1949</a></span><span class="t"><span class="str">        Wrapper around load_and_process_state_dict to allow for in-place processing of the weights.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1950" href="#t1950">1950</a></span><span class="t"><span class="str">        This is useful if using HookedTransformer for training, if we then want to analyse a cleaner</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1951" href="#t1951">1951</a></span><span class="t"><span class="str">        version of the same model.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1952" href="#t1952">1952</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1953" href="#t1953">1953</a></span><span class="t">        <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">state_dict</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1954" href="#t1954">1954</a></span><span class="t">        <span class="key">if</span> <span class="nam">fold_ln</span> <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">num_experts</span> <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">num_experts</span> <span class="op">></span> <span class="num">1</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1954&#x202F;&#x219B;&#x202F;1957</span><span class="annotate long">line 1954 didn't jump to line 1957, because the condition on line 1954 was never true</span></span></p>
-    <p class="pln"><span class="n"><a id="t1955" href="#t1955">1955</a></span><span class="t">            <span class="com"># If we're using MoE, we don't fold the layer norm weights, so we don't need to do any preprocessing</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1956" href="#t1956">1956</a></span><span class="t">            <span class="com"># A warning is already issued in `load_and_process_state_dict`</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1957" href="#t1957">1957</a></span><span class="t">            <span class="key">pass</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1958" href="#t1958">1958</a></span><span class="t">        <span class="key">elif</span> <span class="nam">fold_ln</span> <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="op">==</span> <span class="str">"LN"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1958&#x202F;&#x219B;&#x202F;1969</span><span class="annotate long">line 1958 didn't jump to line 1969, because the condition on line 1958 was never false</span></span></p>
-    <p class="pln"><span class="n"><a id="t1959" href="#t1959">1959</a></span><span class="t">            <span class="com"># If we're folding the LN into the weights, we need to replace all the layernorm layers</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1960" href="#t1960">1960</a></span><span class="t">            <span class="com"># with LayerNormPres, which do not have learnable parameters. This is somewhat hacky,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1961" href="#t1961">1961</a></span><span class="t">            <span class="com"># but it's the easiest way to do it.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1962" href="#t1962">1962</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="op">=</span> <span class="str">"LNPre"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1963" href="#t1963">1963</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">ln_final</span> <span class="op">=</span> <span class="nam">LayerNormPre</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1964" href="#t1964">1964</a></span><span class="t">            <span class="key">for</span> <span class="nam">layer</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1965" href="#t1965">1965</a></span><span class="t">                <span class="nam">layer</span><span class="op">.</span><span class="nam">ln1</span> <span class="op">=</span> <span class="nam">LayerNormPre</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1966" href="#t1966">1966</a></span><span class="t">                <span class="nam">layer</span><span class="op">.</span><span class="nam">ln2</span> <span class="op">=</span> <span class="nam">LayerNormPre</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1967" href="#t1967">1967</a></span><span class="t">                <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">is_layer_norm_activation</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1967&#x202F;&#x219B;&#x202F;1968</span><span class="annotate long">line 1967 didn't jump to line 1968, because the condition on line 1967 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1968" href="#t1968">1968</a></span><span class="t">                    <span class="nam">layer</span><span class="op">.</span><span class="nam">mlp</span><span class="op">.</span><span class="nam">ln</span> <span class="op">=</span> <span class="nam">LayerNormPre</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1969" href="#t1969">1969</a></span><span class="t">        <span class="key">elif</span> <span class="nam">fold_ln</span> <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="op">==</span> <span class="str">"RMS"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1970" href="#t1970">1970</a></span><span class="t">            <span class="com"># We do the same for RMSNorm if used</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1971" href="#t1971">1971</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="op">=</span> <span class="str">"RMSPre"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1972" href="#t1972">1972</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">ln_final</span> <span class="op">=</span> <span class="nam">RMSNormPre</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1973" href="#t1973">1973</a></span><span class="t">            <span class="key">for</span> <span class="nam">layer</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1974" href="#t1974">1974</a></span><span class="t">                <span class="nam">layer</span><span class="op">.</span><span class="nam">ln1</span> <span class="op">=</span> <span class="nam">RMSNormPre</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1975" href="#t1975">1975</a></span><span class="t">                <span class="nam">layer</span><span class="op">.</span><span class="nam">ln2</span> <span class="op">=</span> <span class="nam">RMSNormPre</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1976" href="#t1976">1976</a></span><span class="t">                <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">is_layer_norm_activation</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1977" href="#t1977">1977</a></span><span class="t">                    <span class="nam">layer</span><span class="op">.</span><span class="nam">mlp</span><span class="op">.</span><span class="nam">ln</span> <span class="op">=</span> <span class="nam">RMSNormPre</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1978" href="#t1978">1978</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1979" href="#t1979">1979</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">load_and_process_state_dict</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1980" href="#t1980">1980</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1981" href="#t1981">1981</a></span><span class="t">            <span class="nam">fold_ln</span><span class="op">=</span><span class="nam">fold_ln</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1982" href="#t1982">1982</a></span><span class="t">            <span class="nam">center_writing_weights</span><span class="op">=</span><span class="nam">center_writing_weights</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1983" href="#t1983">1983</a></span><span class="t">            <span class="nam">center_unembed</span><span class="op">=</span><span class="nam">center_unembed</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1984" href="#t1984">1984</a></span><span class="t">            <span class="nam">refactor_factored_attn_matrices</span><span class="op">=</span><span class="nam">refactor_factored_attn_matrices</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1985" href="#t1985">1985</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1986" href="#t1986">1986</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1987" href="#t1987">1987</a></span><span class="t">    <span class="op">@</span><span class="nam">torch</span><span class="op">.</span><span class="nam">inference_mode</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1988" href="#t1988">1988</a></span><span class="t">    <span class="key">def</span> <span class="nam">generate</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1989" href="#t1989">1989</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1990" href="#t1990">1990</a></span><span class="t">        <span class="nam">input</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos"</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="str">""</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1991" href="#t1991">1991</a></span><span class="t">        <span class="nam">max_new_tokens</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">10</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1992" href="#t1992">1992</a></span><span class="t">        <span class="nam">stop_at_eos</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1993" href="#t1993">1993</a></span><span class="t">        <span class="nam">eos_token_id</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1994" href="#t1994">1994</a></span><span class="t">        <span class="nam">do_sample</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1995" href="#t1995">1995</a></span><span class="t">        <span class="nam">top_k</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1996" href="#t1996">1996</a></span><span class="t">        <span class="nam">top_p</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">float</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1997" href="#t1997">1997</a></span><span class="t">        <span class="nam">temperature</span><span class="op">:</span> <span class="nam">float</span> <span class="op">=</span> <span class="num">1.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1998" href="#t1998">1998</a></span><span class="t">        <span class="nam">freq_penalty</span><span class="op">:</span> <span class="nam">float</span> <span class="op">=</span> <span class="num">0.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1999" href="#t1999">1999</a></span><span class="t">        <span class="nam">use_past_kv_cache</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2000" href="#t2000">2000</a></span><span class="t">        <span class="nam">prepend_bos</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">bool</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2001" href="#t2001">2001</a></span><span class="t">        <span class="nam">padding_side</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Literal</span><span class="op">[</span><span class="str">"left"</span><span class="op">,</span> <span class="str">"right"</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2002" href="#t2002">2002</a></span><span class="t">        <span class="nam">return_type</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">str</span><span class="op">]</span> <span class="op">=</span> <span class="str">"input"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2003" href="#t2003">2003</a></span><span class="t">        <span class="nam">verbose</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2004" href="#t2004">2004</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Union</span><span class="op">[</span><span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos_plus_new_tokens"</span><span class="op">]</span><span class="op">,</span> <span class="nam">str</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2005" href="#t2005">2005</a></span><span class="t">        <span class="str">"""Sample Tokens from the Model.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2006" href="#t2006">2006</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2007" href="#t2007">2007</a></span><span class="t"><span class="str">        Sample tokens from the model until the model outputs eos_token or max_new_tokens is reached.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2008" href="#t2008">2008</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2009" href="#t2009">2009</a></span><span class="t"><span class="str">        To avoid fiddling with ragged tensors, if we input a batch of text and some sequences finish</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2010" href="#t2010">2010</a></span><span class="t"><span class="str">        (by producing an EOT token), we keep running the model on the entire batch, but throw away</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2011" href="#t2011">2011</a></span><span class="t"><span class="str">        the output for a finished sequence and just keep adding EOTs to pad.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2012" href="#t2012">2012</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2013" href="#t2013">2013</a></span><span class="t"><span class="str">        This supports entering a single string, but not a list of strings - if the strings don't</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2014" href="#t2014">2014</a></span><span class="t"><span class="str">        tokenize to exactly the same length, this gets messy. If that functionality is needed,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2015" href="#t2015">2015</a></span><span class="t"><span class="str">        convert them to a batch of tokens and input that instead.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2016" href="#t2016">2016</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2017" href="#t2017">2017</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2018" href="#t2018">2018</a></span><span class="t"><span class="str">            input (Union[str, Int[torch.Tensor, "batch pos"])]): Either a batch of tokens ([batch,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2019" href="#t2019">2019</a></span><span class="t"><span class="str">                pos]) or a text string (this will be converted to a batch of tokens with batch size</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2020" href="#t2020">2020</a></span><span class="t"><span class="str">                1).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2021" href="#t2021">2021</a></span><span class="t"><span class="str">            max_new_tokens (int): Maximum number of tokens to generate.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2022" href="#t2022">2022</a></span><span class="t"><span class="str">            stop_at_eos (bool): If True, stop generating tokens when the model outputs eos_token.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2023" href="#t2023">2023</a></span><span class="t"><span class="str">            eos_token_id (Optional[Union[int, Sequence]]): The token ID to use for end</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2024" href="#t2024">2024</a></span><span class="t"><span class="str">                of sentence. If None, use the tokenizer's eos_token_id - required if using</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2025" href="#t2025">2025</a></span><span class="t"><span class="str">                stop_at_eos. It's also possible to provide a list of token IDs (not just the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2026" href="#t2026">2026</a></span><span class="t"><span class="str">                eos_token_id), in which case the generation will stop when any of them are output</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2027" href="#t2027">2027</a></span><span class="t"><span class="str">                (useful e.g. for stable_lm).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2028" href="#t2028">2028</a></span><span class="t"><span class="str">            do_sample (bool): If True, sample from the model's output distribution. Otherwise, use</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2029" href="#t2029">2029</a></span><span class="t"><span class="str">                greedy search (take the max logit each time).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2030" href="#t2030">2030</a></span><span class="t"><span class="str">            top_k (int): Number of tokens to sample from. If None, sample from all tokens.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2031" href="#t2031">2031</a></span><span class="t"><span class="str">            top_p (float): Probability mass to sample from. If 1.0, sample from all tokens. If &lt;1.0,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2032" href="#t2032">2032</a></span><span class="t"><span class="str">                we take the top tokens with cumulative probability >= top_p.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2033" href="#t2033">2033</a></span><span class="t"><span class="str">            temperature (float): Temperature for sampling. Higher values will make the model more</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2034" href="#t2034">2034</a></span><span class="t"><span class="str">                random (limit of temp -> 0 is just taking the top token, limit of temp -> inf is</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2035" href="#t2035">2035</a></span><span class="t"><span class="str">                sampling from a uniform distribution).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2036" href="#t2036">2036</a></span><span class="t"><span class="str">            freq_penalty (float): Frequency penalty for sampling - how much to penalise previous</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2037" href="#t2037">2037</a></span><span class="t"><span class="str">                tokens. Higher values will make the model more random.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2038" href="#t2038">2038</a></span><span class="t"><span class="str">            use_past_kv_cache (bool): If True, create and use cache to speed up generation.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2039" href="#t2039">2039</a></span><span class="t"><span class="str">            prepend_bos (bool, optional): Overrides self.cfg.default_prepend_bos. Whether to prepend</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2040" href="#t2040">2040</a></span><span class="t"><span class="str">                the BOS token to the input (applicable when input is a string). Defaults to None,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2041" href="#t2041">2041</a></span><span class="t"><span class="str">                implying usage of self.cfg.default_prepend_bos (default is True unless specified</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2042" href="#t2042">2042</a></span><span class="t"><span class="str">                otherwise). Pass True or False to override the default.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2043" href="#t2043">2043</a></span><span class="t"><span class="str">            padding_side (Union[Literal["left", "right"], None], optional): Overrides</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2044" href="#t2044">2044</a></span><span class="t"><span class="str">                self.tokenizer.padding_side. Specifies which side to pad when tokenizing multiple</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2045" href="#t2045">2045</a></span><span class="t"><span class="str">                strings of different lengths.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2046" href="#t2046">2046</a></span><span class="t"><span class="str">            return_type (Optional[str]): The type of the output to return - either a string (str),</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2047" href="#t2047">2047</a></span><span class="t"><span class="str">                a tensor of tokens (tensor) or whatever the format of the input was (input).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2048" href="#t2048">2048</a></span><span class="t"><span class="str">            verbose (bool): If True, show tqdm progress bars for generation.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2049" href="#t2049">2049</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2050" href="#t2050">2050</a></span><span class="t"><span class="str">        Returns:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2051" href="#t2051">2051</a></span><span class="t"><span class="str">            outputs (torch.Tensor): [batch, pos + max_new_tokens], generated sequence of new tokens</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2052" href="#t2052">2052</a></span><span class="t"><span class="str">                (by default returns same type as input).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2053" href="#t2053">2053</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2054" href="#t2054">2054</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2055" href="#t2055">2055</a></span><span class="t">        <span class="key">with</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">LocallyOverridenDefaults</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2056" href="#t2056">2056</a></span><span class="t">            <span class="nam">self</span><span class="op">,</span> <span class="nam">prepend_bos</span><span class="op">=</span><span class="nam">prepend_bos</span><span class="op">,</span> <span class="nam">padding_side</span><span class="op">=</span><span class="nam">padding_side</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2057" href="#t2057">2057</a></span><span class="t">        <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2058" href="#t2058">2058</a></span><span class="t">            <span class="key">if</span> <span class="nam">type</span><span class="op">(</span><span class="nam">input</span><span class="op">)</span> <span class="op">==</span> <span class="nam">str</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2059" href="#t2059">2059</a></span><span class="t">                <span class="com"># If text, convert to tokens (batch_size=1)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2060" href="#t2060">2060</a></span><span class="t">                <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2061" href="#t2061">2061</a></span><span class="t">                    <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2062" href="#t2062">2062</a></span><span class="t">                <span class="op">)</span><span class="op">,</span> <span class="str">"Must provide a tokenizer if passing a string to the model"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2063" href="#t2063">2063</a></span><span class="t">                <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">to_tokens</span><span class="op">(</span><span class="nam">input</span><span class="op">,</span> <span class="nam">prepend_bos</span><span class="op">=</span><span class="nam">prepend_bos</span><span class="op">,</span> <span class="nam">padding_side</span><span class="op">=</span><span class="nam">padding_side</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2064" href="#t2064">2064</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2065" href="#t2065">2065</a></span><span class="t">                <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">input</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2066" href="#t2066">2066</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2067" href="#t2067">2067</a></span><span class="t">            <span class="key">if</span> <span class="nam">return_type</span> <span class="op">==</span> <span class="str">"input"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2068" href="#t2068">2068</a></span><span class="t">                <span class="key">if</span> <span class="nam">type</span><span class="op">(</span><span class="nam">input</span><span class="op">)</span> <span class="op">==</span> <span class="nam">str</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2069" href="#t2069">2069</a></span><span class="t">                    <span class="nam">return_type</span> <span class="op">=</span> <span class="str">"str"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2070" href="#t2070">2070</a></span><span class="t">                <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2071" href="#t2071">2071</a></span><span class="t">                    <span class="nam">return_type</span> <span class="op">=</span> <span class="str">"tensor"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2072" href="#t2072">2072</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2073" href="#t2073">2073</a></span><span class="t">            <span class="key">assert</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">tokens</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2074" href="#t2074">2074</a></span><span class="t">            <span class="nam">batch_size</span><span class="op">,</span> <span class="nam">ctx_length</span> <span class="op">=</span> <span class="nam">tokens</span><span class="op">.</span><span class="nam">shape</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2075" href="#t2075">2075</a></span><span class="t">            <span class="nam">device</span> <span class="op">=</span> <span class="nam">devices</span><span class="op">.</span><span class="nam">get_device_for_block_index</span><span class="op">(</span><span class="num">0</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2076" href="#t2076">2076</a></span><span class="t">            <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">tokens</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">device</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2077" href="#t2077">2077</a></span><span class="t">            <span class="key">if</span> <span class="nam">use_past_kv_cache</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2078" href="#t2078">2078</a></span><span class="t">                <span class="nam">past_kv_cache</span> <span class="op">=</span> <span class="nam">HookedTransformerKeyValueCache</span><span class="op">.</span><span class="nam">init_cache</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2079" href="#t2079">2079</a></span><span class="t">                    <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">device</span><span class="op">,</span> <span class="nam">batch_size</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2080" href="#t2080">2080</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2081" href="#t2081">2081</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2082" href="#t2082">2082</a></span><span class="t">                <span class="nam">past_kv_cache</span> <span class="op">=</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2083" href="#t2083">2083</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2084" href="#t2084">2084</a></span><span class="t">            <span class="nam">stop_tokens</span><span class="op">:</span> <span class="nam">List</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="op">[</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2085" href="#t2085">2085</a></span><span class="t">            <span class="nam">eos_token_for_padding</span> <span class="op">=</span> <span class="num">0</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2086" href="#t2086">2086</a></span><span class="t">            <span class="key">assert</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2087" href="#t2087">2087</a></span><span class="t">            <span class="key">if</span> <span class="nam">stop_at_eos</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2088" href="#t2088">2088</a></span><span class="t">                <span class="nam">tokenizer_has_eos_token</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2089" href="#t2089">2089</a></span><span class="t">                    <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span> <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">eos_token_id</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2090" href="#t2090">2090</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2091" href="#t2091">2091</a></span><span class="t">                <span class="key">if</span> <span class="nam">eos_token_id</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2092" href="#t2092">2092</a></span><span class="t">                    <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2093" href="#t2093">2093</a></span><span class="t">                        <span class="nam">tokenizer_has_eos_token</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2094" href="#t2094">2094</a></span><span class="t">                    <span class="op">)</span><span class="op">,</span> <span class="str">"Must pass a eos_token_id if stop_at_eos is True and tokenizer is None or has no eos_token_id"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2095" href="#t2095">2095</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2096" href="#t2096">2096</a></span><span class="t">                    <span class="nam">eos_token_id</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">eos_token_id</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2097" href="#t2097">2097</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2098" href="#t2098">2098</a></span><span class="t">                <span class="key">if</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">eos_token_id</span><span class="op">,</span> <span class="nam">int</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2099" href="#t2099">2099</a></span><span class="t">                    <span class="nam">stop_tokens</span> <span class="op">=</span> <span class="op">[</span><span class="nam">eos_token_id</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2100" href="#t2100">2100</a></span><span class="t">                    <span class="nam">eos_token_for_padding</span> <span class="op">=</span> <span class="nam">eos_token_id</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2101" href="#t2101">2101</a></span><span class="t">                <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2102" href="#t2102">2102</a></span><span class="t">                    <span class="com"># eos_token_id is a Sequence (e.g. list or tuple)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2103" href="#t2103">2103</a></span><span class="t">                    <span class="nam">stop_tokens</span> <span class="op">=</span> <span class="nam">eos_token_id</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2104" href="#t2104">2104</a></span><span class="t">                    <span class="nam">eos_token_for_padding</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2105" href="#t2105">2105</a></span><span class="t">                        <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">eos_token_id</span> <span class="key">if</span> <span class="nam">tokenizer_has_eos_token</span> <span class="key">else</span> <span class="nam">eos_token_id</span><span class="op">[</span><span class="num">0</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2106" href="#t2106">2106</a></span><span class="t">                    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2107" href="#t2107">2107</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2108" href="#t2108">2108</a></span><span class="t">            <span class="com"># An array to track which sequences in the batch have finished.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2109" href="#t2109">2109</a></span><span class="t">            <span class="nam">finished_sequences</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">zeros</span><span class="op">(</span><span class="nam">batch_size</span><span class="op">,</span> <span class="nam">dtype</span><span class="op">=</span><span class="nam">torch</span><span class="op">.</span><span class="nam">bool</span><span class="op">,</span> <span class="nam">device</span><span class="op">=</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">device</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1934" href="#t1934">1934</a></span><span class="t">        <span class="key">assert</span> <span class="key">not</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">attn_only</span><span class="op">,</span> <span class="str">"Can't use hook_mlp_in with attn_only model"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1935" href="#t1935">1935</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">use_hook_mlp_in</span> <span class="op">=</span> <span class="nam">use_hook_mlp_in</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1936" href="#t1936">1936</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1937" href="#t1937">1937</a></span><span class="t">    <span class="key">def</span> <span class="nam">set_use_attn_in</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">use_attn_in</span><span class="op">:</span> <span class="nam">bool</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1938" href="#t1938">1938</a></span><span class="t">        <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1939" href="#t1939">1939</a></span><span class="t"><span class="str">        Toggles whether to allow editing of inputs to each attention head.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1940" href="#t1940">1940</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1941" href="#t1941">1941</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">use_attn_in</span> <span class="op">=</span> <span class="nam">use_attn_in</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1942" href="#t1942">1942</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1943" href="#t1943">1943</a></span><span class="t">    <span class="key">def</span> <span class="nam">process_weights_</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1944" href="#t1944">1944</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1945" href="#t1945">1945</a></span><span class="t">        <span class="nam">fold_ln</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1946" href="#t1946">1946</a></span><span class="t">        <span class="nam">center_writing_weights</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1947" href="#t1947">1947</a></span><span class="t">        <span class="nam">center_unembed</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1948" href="#t1948">1948</a></span><span class="t">        <span class="nam">refactor_factored_attn_matrices</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1949" href="#t1949">1949</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1950" href="#t1950">1950</a></span><span class="t">        <span class="str">"""Wrapper around `load_and_process_state_dict`.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1951" href="#t1951">1951</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1952" href="#t1952">1952</a></span><span class="t"><span class="str">        Wrapper around load_and_process_state_dict to allow for in-place processing of the weights.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1953" href="#t1953">1953</a></span><span class="t"><span class="str">        This is useful if using HookedTransformer for training, if we then want to analyse a cleaner</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1954" href="#t1954">1954</a></span><span class="t"><span class="str">        version of the same model.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1955" href="#t1955">1955</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1956" href="#t1956">1956</a></span><span class="t">        <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">state_dict</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1957" href="#t1957">1957</a></span><span class="t">        <span class="key">if</span> <span class="nam">fold_ln</span> <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">num_experts</span> <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">num_experts</span> <span class="op">></span> <span class="num">1</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1957&#x202F;&#x219B;&#x202F;1960</span><span class="annotate long">line 1957 didn't jump to line 1960, because the condition on line 1957 was never true</span></span></p>
+    <p class="pln"><span class="n"><a id="t1958" href="#t1958">1958</a></span><span class="t">            <span class="com"># If we're using MoE, we don't fold the layer norm weights, so we don't need to do any preprocessing</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1959" href="#t1959">1959</a></span><span class="t">            <span class="com"># A warning is already issued in `load_and_process_state_dict`</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1960" href="#t1960">1960</a></span><span class="t">            <span class="key">pass</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1961" href="#t1961">1961</a></span><span class="t">        <span class="key">elif</span> <span class="nam">fold_ln</span> <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="op">==</span> <span class="str">"LN"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1961&#x202F;&#x219B;&#x202F;1972</span><span class="annotate long">line 1961 didn't jump to line 1972, because the condition on line 1961 was never false</span></span></p>
+    <p class="pln"><span class="n"><a id="t1962" href="#t1962">1962</a></span><span class="t">            <span class="com"># If we're folding the LN into the weights, we need to replace all the layernorm layers</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1963" href="#t1963">1963</a></span><span class="t">            <span class="com"># with LayerNormPres, which do not have learnable parameters. This is somewhat hacky,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1964" href="#t1964">1964</a></span><span class="t">            <span class="com"># but it's the easiest way to do it.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1965" href="#t1965">1965</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="op">=</span> <span class="str">"LNPre"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1966" href="#t1966">1966</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">ln_final</span> <span class="op">=</span> <span class="nam">LayerNormPre</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1967" href="#t1967">1967</a></span><span class="t">            <span class="key">for</span> <span class="nam">layer</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1968" href="#t1968">1968</a></span><span class="t">                <span class="nam">layer</span><span class="op">.</span><span class="nam">ln1</span> <span class="op">=</span> <span class="nam">LayerNormPre</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1969" href="#t1969">1969</a></span><span class="t">                <span class="nam">layer</span><span class="op">.</span><span class="nam">ln2</span> <span class="op">=</span> <span class="nam">LayerNormPre</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1970" href="#t1970">1970</a></span><span class="t">                <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">is_layer_norm_activation</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1970&#x202F;&#x219B;&#x202F;1971</span><span class="annotate long">line 1970 didn't jump to line 1971, because the condition on line 1970 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1971" href="#t1971">1971</a></span><span class="t">                    <span class="nam">layer</span><span class="op">.</span><span class="nam">mlp</span><span class="op">.</span><span class="nam">ln</span> <span class="op">=</span> <span class="nam">LayerNormPre</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1972" href="#t1972">1972</a></span><span class="t">        <span class="key">elif</span> <span class="nam">fold_ln</span> <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="op">==</span> <span class="str">"RMS"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1973" href="#t1973">1973</a></span><span class="t">            <span class="com"># We do the same for RMSNorm if used</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1974" href="#t1974">1974</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">normalization_type</span> <span class="op">=</span> <span class="str">"RMSPre"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1975" href="#t1975">1975</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">ln_final</span> <span class="op">=</span> <span class="nam">RMSNormPre</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1976" href="#t1976">1976</a></span><span class="t">            <span class="key">for</span> <span class="nam">layer</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1977" href="#t1977">1977</a></span><span class="t">                <span class="nam">layer</span><span class="op">.</span><span class="nam">ln1</span> <span class="op">=</span> <span class="nam">RMSNormPre</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1978" href="#t1978">1978</a></span><span class="t">                <span class="nam">layer</span><span class="op">.</span><span class="nam">ln2</span> <span class="op">=</span> <span class="nam">RMSNormPre</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1979" href="#t1979">1979</a></span><span class="t">                <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">is_layer_norm_activation</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1980" href="#t1980">1980</a></span><span class="t">                    <span class="nam">layer</span><span class="op">.</span><span class="nam">mlp</span><span class="op">.</span><span class="nam">ln</span> <span class="op">=</span> <span class="nam">RMSNormPre</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1981" href="#t1981">1981</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1982" href="#t1982">1982</a></span><span class="t">        <span class="nam">self</span><span class="op">.</span><span class="nam">load_and_process_state_dict</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1983" href="#t1983">1983</a></span><span class="t">            <span class="nam">state_dict</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1984" href="#t1984">1984</a></span><span class="t">            <span class="nam">fold_ln</span><span class="op">=</span><span class="nam">fold_ln</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1985" href="#t1985">1985</a></span><span class="t">            <span class="nam">center_writing_weights</span><span class="op">=</span><span class="nam">center_writing_weights</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1986" href="#t1986">1986</a></span><span class="t">            <span class="nam">center_unembed</span><span class="op">=</span><span class="nam">center_unembed</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1987" href="#t1987">1987</a></span><span class="t">            <span class="nam">refactor_factored_attn_matrices</span><span class="op">=</span><span class="nam">refactor_factored_attn_matrices</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1988" href="#t1988">1988</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1989" href="#t1989">1989</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1990" href="#t1990">1990</a></span><span class="t">    <span class="op">@</span><span class="nam">torch</span><span class="op">.</span><span class="nam">inference_mode</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1991" href="#t1991">1991</a></span><span class="t">    <span class="key">def</span> <span class="nam">generate</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1992" href="#t1992">1992</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1993" href="#t1993">1993</a></span><span class="t">        <span class="nam">input</span><span class="op">:</span> <span class="nam">Union</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos"</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="str">""</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1994" href="#t1994">1994</a></span><span class="t">        <span class="nam">max_new_tokens</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">10</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1995" href="#t1995">1995</a></span><span class="t">        <span class="nam">stop_at_eos</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1996" href="#t1996">1996</a></span><span class="t">        <span class="nam">eos_token_id</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1997" href="#t1997">1997</a></span><span class="t">        <span class="nam">do_sample</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1998" href="#t1998">1998</a></span><span class="t">        <span class="nam">top_k</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1999" href="#t1999">1999</a></span><span class="t">        <span class="nam">top_p</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">float</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2000" href="#t2000">2000</a></span><span class="t">        <span class="nam">temperature</span><span class="op">:</span> <span class="nam">float</span> <span class="op">=</span> <span class="num">1.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2001" href="#t2001">2001</a></span><span class="t">        <span class="nam">freq_penalty</span><span class="op">:</span> <span class="nam">float</span> <span class="op">=</span> <span class="num">0.0</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2002" href="#t2002">2002</a></span><span class="t">        <span class="nam">use_past_kv_cache</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2003" href="#t2003">2003</a></span><span class="t">        <span class="nam">prepend_bos</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">bool</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2004" href="#t2004">2004</a></span><span class="t">        <span class="nam">padding_side</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Literal</span><span class="op">[</span><span class="str">"left"</span><span class="op">,</span> <span class="str">"right"</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2005" href="#t2005">2005</a></span><span class="t">        <span class="nam">return_type</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">str</span><span class="op">]</span> <span class="op">=</span> <span class="str">"input"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2006" href="#t2006">2006</a></span><span class="t">        <span class="nam">verbose</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2007" href="#t2007">2007</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Union</span><span class="op">[</span><span class="nam">Int</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"batch pos_plus_new_tokens"</span><span class="op">]</span><span class="op">,</span> <span class="nam">str</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2008" href="#t2008">2008</a></span><span class="t">        <span class="str">"""Sample Tokens from the Model.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2009" href="#t2009">2009</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2010" href="#t2010">2010</a></span><span class="t"><span class="str">        Sample tokens from the model until the model outputs eos_token or max_new_tokens is reached.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2011" href="#t2011">2011</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2012" href="#t2012">2012</a></span><span class="t"><span class="str">        To avoid fiddling with ragged tensors, if we input a batch of text and some sequences finish</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2013" href="#t2013">2013</a></span><span class="t"><span class="str">        (by producing an EOT token), we keep running the model on the entire batch, but throw away</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2014" href="#t2014">2014</a></span><span class="t"><span class="str">        the output for a finished sequence and just keep adding EOTs to pad.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2015" href="#t2015">2015</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2016" href="#t2016">2016</a></span><span class="t"><span class="str">        This supports entering a single string, but not a list of strings - if the strings don't</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2017" href="#t2017">2017</a></span><span class="t"><span class="str">        tokenize to exactly the same length, this gets messy. If that functionality is needed,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2018" href="#t2018">2018</a></span><span class="t"><span class="str">        convert them to a batch of tokens and input that instead.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2019" href="#t2019">2019</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2020" href="#t2020">2020</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2021" href="#t2021">2021</a></span><span class="t"><span class="str">            input (Union[str, Int[torch.Tensor, "batch pos"])]): Either a batch of tokens ([batch,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2022" href="#t2022">2022</a></span><span class="t"><span class="str">                pos]) or a text string (this will be converted to a batch of tokens with batch size</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2023" href="#t2023">2023</a></span><span class="t"><span class="str">                1).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2024" href="#t2024">2024</a></span><span class="t"><span class="str">            max_new_tokens (int): Maximum number of tokens to generate.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2025" href="#t2025">2025</a></span><span class="t"><span class="str">            stop_at_eos (bool): If True, stop generating tokens when the model outputs eos_token.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2026" href="#t2026">2026</a></span><span class="t"><span class="str">            eos_token_id (Optional[Union[int, Sequence]]): The token ID to use for end</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2027" href="#t2027">2027</a></span><span class="t"><span class="str">                of sentence. If None, use the tokenizer's eos_token_id - required if using</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2028" href="#t2028">2028</a></span><span class="t"><span class="str">                stop_at_eos. It's also possible to provide a list of token IDs (not just the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2029" href="#t2029">2029</a></span><span class="t"><span class="str">                eos_token_id), in which case the generation will stop when any of them are output</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2030" href="#t2030">2030</a></span><span class="t"><span class="str">                (useful e.g. for stable_lm).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2031" href="#t2031">2031</a></span><span class="t"><span class="str">            do_sample (bool): If True, sample from the model's output distribution. Otherwise, use</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2032" href="#t2032">2032</a></span><span class="t"><span class="str">                greedy search (take the max logit each time).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2033" href="#t2033">2033</a></span><span class="t"><span class="str">            top_k (int): Number of tokens to sample from. If None, sample from all tokens.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2034" href="#t2034">2034</a></span><span class="t"><span class="str">            top_p (float): Probability mass to sample from. If 1.0, sample from all tokens. If &lt;1.0,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2035" href="#t2035">2035</a></span><span class="t"><span class="str">                we take the top tokens with cumulative probability >= top_p.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2036" href="#t2036">2036</a></span><span class="t"><span class="str">            temperature (float): Temperature for sampling. Higher values will make the model more</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2037" href="#t2037">2037</a></span><span class="t"><span class="str">                random (limit of temp -> 0 is just taking the top token, limit of temp -> inf is</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2038" href="#t2038">2038</a></span><span class="t"><span class="str">                sampling from a uniform distribution).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2039" href="#t2039">2039</a></span><span class="t"><span class="str">            freq_penalty (float): Frequency penalty for sampling - how much to penalise previous</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2040" href="#t2040">2040</a></span><span class="t"><span class="str">                tokens. Higher values will make the model more random.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2041" href="#t2041">2041</a></span><span class="t"><span class="str">            use_past_kv_cache (bool): If True, create and use cache to speed up generation.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2042" href="#t2042">2042</a></span><span class="t"><span class="str">            prepend_bos (bool, optional): Overrides self.cfg.default_prepend_bos. Whether to prepend</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2043" href="#t2043">2043</a></span><span class="t"><span class="str">                the BOS token to the input (applicable when input is a string). Defaults to None,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2044" href="#t2044">2044</a></span><span class="t"><span class="str">                implying usage of self.cfg.default_prepend_bos (default is True unless specified</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2045" href="#t2045">2045</a></span><span class="t"><span class="str">                otherwise). Pass True or False to override the default.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2046" href="#t2046">2046</a></span><span class="t"><span class="str">            padding_side (Union[Literal["left", "right"], None], optional): Overrides</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2047" href="#t2047">2047</a></span><span class="t"><span class="str">                self.tokenizer.padding_side. Specifies which side to pad when tokenizing multiple</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2048" href="#t2048">2048</a></span><span class="t"><span class="str">                strings of different lengths.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2049" href="#t2049">2049</a></span><span class="t"><span class="str">            return_type (Optional[str]): The type of the output to return - either a string (str),</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2050" href="#t2050">2050</a></span><span class="t"><span class="str">                a tensor of tokens (tensor) or whatever the format of the input was (input).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2051" href="#t2051">2051</a></span><span class="t"><span class="str">            verbose (bool): If True, show tqdm progress bars for generation.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2052" href="#t2052">2052</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2053" href="#t2053">2053</a></span><span class="t"><span class="str">        Returns:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2054" href="#t2054">2054</a></span><span class="t"><span class="str">            outputs (torch.Tensor): [batch, pos + max_new_tokens], generated sequence of new tokens</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2055" href="#t2055">2055</a></span><span class="t"><span class="str">                (by default returns same type as input).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2056" href="#t2056">2056</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2057" href="#t2057">2057</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2058" href="#t2058">2058</a></span><span class="t">        <span class="key">with</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">LocallyOverridenDefaults</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2059" href="#t2059">2059</a></span><span class="t">            <span class="nam">self</span><span class="op">,</span> <span class="nam">prepend_bos</span><span class="op">=</span><span class="nam">prepend_bos</span><span class="op">,</span> <span class="nam">padding_side</span><span class="op">=</span><span class="nam">padding_side</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2060" href="#t2060">2060</a></span><span class="t">        <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2061" href="#t2061">2061</a></span><span class="t">            <span class="key">if</span> <span class="nam">type</span><span class="op">(</span><span class="nam">input</span><span class="op">)</span> <span class="op">==</span> <span class="nam">str</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2062" href="#t2062">2062</a></span><span class="t">                <span class="com"># If text, convert to tokens (batch_size=1)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2063" href="#t2063">2063</a></span><span class="t">                <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2064" href="#t2064">2064</a></span><span class="t">                    <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2065" href="#t2065">2065</a></span><span class="t">                <span class="op">)</span><span class="op">,</span> <span class="str">"Must provide a tokenizer if passing a string to the model"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2066" href="#t2066">2066</a></span><span class="t">                <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">to_tokens</span><span class="op">(</span><span class="nam">input</span><span class="op">,</span> <span class="nam">prepend_bos</span><span class="op">=</span><span class="nam">prepend_bos</span><span class="op">,</span> <span class="nam">padding_side</span><span class="op">=</span><span class="nam">padding_side</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2067" href="#t2067">2067</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2068" href="#t2068">2068</a></span><span class="t">                <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">input</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2069" href="#t2069">2069</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2070" href="#t2070">2070</a></span><span class="t">            <span class="key">if</span> <span class="nam">return_type</span> <span class="op">==</span> <span class="str">"input"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2071" href="#t2071">2071</a></span><span class="t">                <span class="key">if</span> <span class="nam">type</span><span class="op">(</span><span class="nam">input</span><span class="op">)</span> <span class="op">==</span> <span class="nam">str</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2072" href="#t2072">2072</a></span><span class="t">                    <span class="nam">return_type</span> <span class="op">=</span> <span class="str">"str"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2073" href="#t2073">2073</a></span><span class="t">                <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2074" href="#t2074">2074</a></span><span class="t">                    <span class="nam">return_type</span> <span class="op">=</span> <span class="str">"tensor"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2075" href="#t2075">2075</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2076" href="#t2076">2076</a></span><span class="t">            <span class="key">assert</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">tokens</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2077" href="#t2077">2077</a></span><span class="t">            <span class="nam">batch_size</span><span class="op">,</span> <span class="nam">ctx_length</span> <span class="op">=</span> <span class="nam">tokens</span><span class="op">.</span><span class="nam">shape</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2078" href="#t2078">2078</a></span><span class="t">            <span class="nam">device</span> <span class="op">=</span> <span class="nam">devices</span><span class="op">.</span><span class="nam">get_device_for_block_index</span><span class="op">(</span><span class="num">0</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2079" href="#t2079">2079</a></span><span class="t">            <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">tokens</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">device</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2080" href="#t2080">2080</a></span><span class="t">            <span class="key">if</span> <span class="nam">use_past_kv_cache</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2081" href="#t2081">2081</a></span><span class="t">                <span class="nam">past_kv_cache</span> <span class="op">=</span> <span class="nam">HookedTransformerKeyValueCache</span><span class="op">.</span><span class="nam">init_cache</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2082" href="#t2082">2082</a></span><span class="t">                    <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">device</span><span class="op">,</span> <span class="nam">batch_size</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2083" href="#t2083">2083</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2084" href="#t2084">2084</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2085" href="#t2085">2085</a></span><span class="t">                <span class="nam">past_kv_cache</span> <span class="op">=</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2086" href="#t2086">2086</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2087" href="#t2087">2087</a></span><span class="t">            <span class="nam">stop_tokens</span><span class="op">:</span> <span class="nam">List</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="op">[</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2088" href="#t2088">2088</a></span><span class="t">            <span class="nam">eos_token_for_padding</span> <span class="op">=</span> <span class="num">0</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2089" href="#t2089">2089</a></span><span class="t">            <span class="key">assert</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2090" href="#t2090">2090</a></span><span class="t">            <span class="key">if</span> <span class="nam">stop_at_eos</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2091" href="#t2091">2091</a></span><span class="t">                <span class="nam">tokenizer_has_eos_token</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2092" href="#t2092">2092</a></span><span class="t">                    <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span> <span class="key">and</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">eos_token_id</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2093" href="#t2093">2093</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2094" href="#t2094">2094</a></span><span class="t">                <span class="key">if</span> <span class="nam">eos_token_id</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2095" href="#t2095">2095</a></span><span class="t">                    <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2096" href="#t2096">2096</a></span><span class="t">                        <span class="nam">tokenizer_has_eos_token</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2097" href="#t2097">2097</a></span><span class="t">                    <span class="op">)</span><span class="op">,</span> <span class="str">"Must pass a eos_token_id if stop_at_eos is True and tokenizer is None or has no eos_token_id"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2098" href="#t2098">2098</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2099" href="#t2099">2099</a></span><span class="t">                    <span class="nam">eos_token_id</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">eos_token_id</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2100" href="#t2100">2100</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2101" href="#t2101">2101</a></span><span class="t">                <span class="key">if</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">eos_token_id</span><span class="op">,</span> <span class="nam">int</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2102" href="#t2102">2102</a></span><span class="t">                    <span class="nam">stop_tokens</span> <span class="op">=</span> <span class="op">[</span><span class="nam">eos_token_id</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2103" href="#t2103">2103</a></span><span class="t">                    <span class="nam">eos_token_for_padding</span> <span class="op">=</span> <span class="nam">eos_token_id</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2104" href="#t2104">2104</a></span><span class="t">                <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2105" href="#t2105">2105</a></span><span class="t">                    <span class="com"># eos_token_id is a Sequence (e.g. list or tuple)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2106" href="#t2106">2106</a></span><span class="t">                    <span class="nam">stop_tokens</span> <span class="op">=</span> <span class="nam">eos_token_id</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2107" href="#t2107">2107</a></span><span class="t">                    <span class="nam">eos_token_for_padding</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2108" href="#t2108">2108</a></span><span class="t">                        <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">eos_token_id</span> <span class="key">if</span> <span class="nam">tokenizer_has_eos_token</span> <span class="key">else</span> <span class="nam">eos_token_id</span><span class="op">[</span><span class="num">0</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2109" href="#t2109">2109</a></span><span class="t">                    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t2110" href="#t2110">2110</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2111" href="#t2111">2111</a></span><span class="t">            <span class="com"># Currently nothing in HookedTransformer changes with eval, but this is here in case</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2112" href="#t2112">2112</a></span><span class="t">            <span class="com"># that changes in the future.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2113" href="#t2113">2113</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">eval</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2114" href="#t2114">2114</a></span><span class="t">            <span class="key">for</span> <span class="nam">index</span> <span class="key">in</span> <span class="nam">tqdm</span><span class="op">.</span><span class="nam">tqdm</span><span class="op">(</span><span class="nam">range</span><span class="op">(</span><span class="nam">max_new_tokens</span><span class="op">)</span><span class="op">,</span> <span class="nam">disable</span><span class="op">=</span><span class="key">not</span> <span class="nam">verbose</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2115" href="#t2115">2115</a></span><span class="t">                <span class="com"># While generating, we keep generating logits, throw away all but the final logits,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2116" href="#t2116">2116</a></span><span class="t">                <span class="com"># and then use those logits to sample from the distribution We keep adding the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2117" href="#t2117">2117</a></span><span class="t">                <span class="com"># sampled tokens to the end of tokens.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2118" href="#t2118">2118</a></span><span class="t">                <span class="key">if</span> <span class="nam">use_past_kv_cache</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2119" href="#t2119">2119</a></span><span class="t">                    <span class="com"># We just take the final tokens, as a [batch, 1] tensor</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2120" href="#t2120">2120</a></span><span class="t">                    <span class="key">if</span> <span class="nam">index</span> <span class="op">></span> <span class="num">0</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2121" href="#t2121">2121</a></span><span class="t">                        <span class="nam">logits</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">forward</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2122" href="#t2122">2122</a></span><span class="t">                            <span class="nam">tokens</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="op">-</span><span class="num">1</span><span class="op">:</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2123" href="#t2123">2123</a></span><span class="t">                            <span class="nam">return_type</span><span class="op">=</span><span class="str">"logits"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2124" href="#t2124">2124</a></span><span class="t">                            <span class="nam">prepend_bos</span><span class="op">=</span><span class="nam">prepend_bos</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2125" href="#t2125">2125</a></span><span class="t">                            <span class="nam">padding_side</span><span class="op">=</span><span class="nam">padding_side</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2126" href="#t2126">2126</a></span><span class="t">                            <span class="nam">past_kv_cache</span><span class="op">=</span><span class="nam">past_kv_cache</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2127" href="#t2127">2127</a></span><span class="t">                        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2128" href="#t2128">2128</a></span><span class="t">                    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2129" href="#t2129">2129</a></span><span class="t">                        <span class="nam">logits</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">forward</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2130" href="#t2130">2130</a></span><span class="t">                            <span class="nam">tokens</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2131" href="#t2131">2131</a></span><span class="t">                            <span class="nam">return_type</span><span class="op">=</span><span class="str">"logits"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2132" href="#t2132">2132</a></span><span class="t">                            <span class="nam">prepend_bos</span><span class="op">=</span><span class="nam">prepend_bos</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2133" href="#t2133">2133</a></span><span class="t">                            <span class="nam">padding_side</span><span class="op">=</span><span class="nam">padding_side</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2134" href="#t2134">2134</a></span><span class="t">                            <span class="nam">past_kv_cache</span><span class="op">=</span><span class="nam">past_kv_cache</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2135" href="#t2135">2135</a></span><span class="t">                        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2136" href="#t2136">2136</a></span><span class="t">                <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2137" href="#t2137">2137</a></span><span class="t">                    <span class="com"># We input the entire sequence, as a [batch, pos] tensor, since we aren't using</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2138" href="#t2138">2138</a></span><span class="t">                    <span class="com"># the cache.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2139" href="#t2139">2139</a></span><span class="t">                    <span class="nam">logits</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">forward</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2140" href="#t2140">2140</a></span><span class="t">                        <span class="nam">tokens</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2141" href="#t2141">2141</a></span><span class="t">                        <span class="nam">return_type</span><span class="op">=</span><span class="str">"logits"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2142" href="#t2142">2142</a></span><span class="t">                        <span class="nam">prepend_bos</span><span class="op">=</span><span class="nam">prepend_bos</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2143" href="#t2143">2143</a></span><span class="t">                        <span class="nam">padding_side</span><span class="op">=</span><span class="nam">padding_side</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2144" href="#t2144">2144</a></span><span class="t">                    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2145" href="#t2145">2145</a></span><span class="t">                <span class="nam">final_logits</span> <span class="op">=</span> <span class="nam">logits</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="op">-</span><span class="num">1</span><span class="op">,</span> <span class="op">:</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2146" href="#t2146">2146</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2147" href="#t2147">2147</a></span><span class="t">                <span class="key">if</span> <span class="nam">do_sample</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2148" href="#t2148">2148</a></span><span class="t">                    <span class="nam">sampled_tokens</span> <span class="op">=</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">sample_logits</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2149" href="#t2149">2149</a></span><span class="t">                        <span class="nam">final_logits</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2150" href="#t2150">2150</a></span><span class="t">                        <span class="nam">top_k</span><span class="op">=</span><span class="nam">top_k</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2151" href="#t2151">2151</a></span><span class="t">                        <span class="nam">top_p</span><span class="op">=</span><span class="nam">top_p</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2152" href="#t2152">2152</a></span><span class="t">                        <span class="nam">temperature</span><span class="op">=</span><span class="nam">temperature</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2153" href="#t2153">2153</a></span><span class="t">                        <span class="nam">freq_penalty</span><span class="op">=</span><span class="nam">freq_penalty</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2154" href="#t2154">2154</a></span><span class="t">                        <span class="nam">tokens</span><span class="op">=</span><span class="nam">tokens</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2155" href="#t2155">2155</a></span><span class="t">                    <span class="op">)</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">devices</span><span class="op">.</span><span class="nam">get_device_for_block_index</span><span class="op">(</span><span class="num">0</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2156" href="#t2156">2156</a></span><span class="t">                <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2157" href="#t2157">2157</a></span><span class="t">                    <span class="nam">sampled_tokens</span> <span class="op">=</span> <span class="nam">final_logits</span><span class="op">.</span><span class="nam">argmax</span><span class="op">(</span><span class="op">-</span><span class="num">1</span><span class="op">)</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2158" href="#t2158">2158</a></span><span class="t">                        <span class="nam">devices</span><span class="op">.</span><span class="nam">get_device_for_block_index</span><span class="op">(</span><span class="num">0</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2159" href="#t2159">2159</a></span><span class="t">                    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2160" href="#t2160">2160</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2161" href="#t2161">2161</a></span><span class="t">                <span class="key">if</span> <span class="nam">stop_at_eos</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2162" href="#t2162">2162</a></span><span class="t">                    <span class="com"># For all unfinished sequences, add on the next token. If a sequence was</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2163" href="#t2163">2163</a></span><span class="t">                    <span class="com"># finished, throw away the generated token and add eos_token_for_padding</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2164" href="#t2164">2164</a></span><span class="t">                    <span class="com"># instead.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2165" href="#t2165">2165</a></span><span class="t">                    <span class="nam">sampled_tokens</span><span class="op">[</span><span class="nam">finished_sequences</span><span class="op">]</span> <span class="op">=</span> <span class="nam">eos_token_for_padding</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2166" href="#t2166">2166</a></span><span class="t">                    <span class="nam">finished_sequences</span><span class="op">.</span><span class="nam">logical_or_</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2167" href="#t2167">2167</a></span><span class="t">                        <span class="nam">torch</span><span class="op">.</span><span class="nam">isin</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2168" href="#t2168">2168</a></span><span class="t">                            <span class="nam">sampled_tokens</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">device</span><span class="op">)</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2169" href="#t2169">2169</a></span><span class="t">                            <span class="nam">torch</span><span class="op">.</span><span class="nam">tensor</span><span class="op">(</span><span class="nam">stop_tokens</span><span class="op">)</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">device</span><span class="op">)</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2170" href="#t2170">2170</a></span><span class="t">                        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2171" href="#t2171">2171</a></span><span class="t">                    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2172" href="#t2172">2172</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2173" href="#t2173">2173</a></span><span class="t">                <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">cat</span><span class="op">(</span><span class="op">[</span><span class="nam">tokens</span><span class="op">,</span> <span class="nam">sampled_tokens</span><span class="op">.</span><span class="nam">unsqueeze</span><span class="op">(</span><span class="op">-</span><span class="num">1</span><span class="op">)</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="op">-</span><span class="num">1</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2174" href="#t2174">2174</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2175" href="#t2175">2175</a></span><span class="t">                <span class="key">if</span> <span class="nam">stop_at_eos</span> <span class="key">and</span> <span class="nam">finished_sequences</span><span class="op">.</span><span class="nam">all</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2176" href="#t2176">2176</a></span><span class="t">                    <span class="key">break</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2111" href="#t2111">2111</a></span><span class="t">            <span class="com"># An array to track which sequences in the batch have finished.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2112" href="#t2112">2112</a></span><span class="t">            <span class="nam">finished_sequences</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">zeros</span><span class="op">(</span><span class="nam">batch_size</span><span class="op">,</span> <span class="nam">dtype</span><span class="op">=</span><span class="nam">torch</span><span class="op">.</span><span class="nam">bool</span><span class="op">,</span> <span class="nam">device</span><span class="op">=</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">device</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2113" href="#t2113">2113</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2114" href="#t2114">2114</a></span><span class="t">            <span class="com"># Currently nothing in HookedTransformer changes with eval, but this is here in case</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2115" href="#t2115">2115</a></span><span class="t">            <span class="com"># that changes in the future.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2116" href="#t2116">2116</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">eval</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2117" href="#t2117">2117</a></span><span class="t">            <span class="key">for</span> <span class="nam">index</span> <span class="key">in</span> <span class="nam">tqdm</span><span class="op">.</span><span class="nam">tqdm</span><span class="op">(</span><span class="nam">range</span><span class="op">(</span><span class="nam">max_new_tokens</span><span class="op">)</span><span class="op">,</span> <span class="nam">disable</span><span class="op">=</span><span class="key">not</span> <span class="nam">verbose</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2118" href="#t2118">2118</a></span><span class="t">                <span class="com"># While generating, we keep generating logits, throw away all but the final logits,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2119" href="#t2119">2119</a></span><span class="t">                <span class="com"># and then use those logits to sample from the distribution We keep adding the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2120" href="#t2120">2120</a></span><span class="t">                <span class="com"># sampled tokens to the end of tokens.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2121" href="#t2121">2121</a></span><span class="t">                <span class="key">if</span> <span class="nam">use_past_kv_cache</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2122" href="#t2122">2122</a></span><span class="t">                    <span class="com"># We just take the final tokens, as a [batch, 1] tensor</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2123" href="#t2123">2123</a></span><span class="t">                    <span class="key">if</span> <span class="nam">index</span> <span class="op">></span> <span class="num">0</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2124" href="#t2124">2124</a></span><span class="t">                        <span class="nam">logits</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">forward</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2125" href="#t2125">2125</a></span><span class="t">                            <span class="nam">tokens</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="op">-</span><span class="num">1</span><span class="op">:</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2126" href="#t2126">2126</a></span><span class="t">                            <span class="nam">return_type</span><span class="op">=</span><span class="str">"logits"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2127" href="#t2127">2127</a></span><span class="t">                            <span class="nam">prepend_bos</span><span class="op">=</span><span class="nam">prepend_bos</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2128" href="#t2128">2128</a></span><span class="t">                            <span class="nam">padding_side</span><span class="op">=</span><span class="nam">padding_side</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2129" href="#t2129">2129</a></span><span class="t">                            <span class="nam">past_kv_cache</span><span class="op">=</span><span class="nam">past_kv_cache</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2130" href="#t2130">2130</a></span><span class="t">                        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2131" href="#t2131">2131</a></span><span class="t">                    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2132" href="#t2132">2132</a></span><span class="t">                        <span class="nam">logits</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">forward</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2133" href="#t2133">2133</a></span><span class="t">                            <span class="nam">tokens</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2134" href="#t2134">2134</a></span><span class="t">                            <span class="nam">return_type</span><span class="op">=</span><span class="str">"logits"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2135" href="#t2135">2135</a></span><span class="t">                            <span class="nam">prepend_bos</span><span class="op">=</span><span class="nam">prepend_bos</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2136" href="#t2136">2136</a></span><span class="t">                            <span class="nam">padding_side</span><span class="op">=</span><span class="nam">padding_side</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2137" href="#t2137">2137</a></span><span class="t">                            <span class="nam">past_kv_cache</span><span class="op">=</span><span class="nam">past_kv_cache</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2138" href="#t2138">2138</a></span><span class="t">                        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2139" href="#t2139">2139</a></span><span class="t">                <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2140" href="#t2140">2140</a></span><span class="t">                    <span class="com"># We input the entire sequence, as a [batch, pos] tensor, since we aren't using</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2141" href="#t2141">2141</a></span><span class="t">                    <span class="com"># the cache.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2142" href="#t2142">2142</a></span><span class="t">                    <span class="nam">logits</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">forward</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2143" href="#t2143">2143</a></span><span class="t">                        <span class="nam">tokens</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2144" href="#t2144">2144</a></span><span class="t">                        <span class="nam">return_type</span><span class="op">=</span><span class="str">"logits"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2145" href="#t2145">2145</a></span><span class="t">                        <span class="nam">prepend_bos</span><span class="op">=</span><span class="nam">prepend_bos</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2146" href="#t2146">2146</a></span><span class="t">                        <span class="nam">padding_side</span><span class="op">=</span><span class="nam">padding_side</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2147" href="#t2147">2147</a></span><span class="t">                    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2148" href="#t2148">2148</a></span><span class="t">                <span class="nam">final_logits</span> <span class="op">=</span> <span class="nam">logits</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="op">-</span><span class="num">1</span><span class="op">,</span> <span class="op">:</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2149" href="#t2149">2149</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2150" href="#t2150">2150</a></span><span class="t">                <span class="key">if</span> <span class="nam">do_sample</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2151" href="#t2151">2151</a></span><span class="t">                    <span class="nam">sampled_tokens</span> <span class="op">=</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">sample_logits</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2152" href="#t2152">2152</a></span><span class="t">                        <span class="nam">final_logits</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2153" href="#t2153">2153</a></span><span class="t">                        <span class="nam">top_k</span><span class="op">=</span><span class="nam">top_k</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2154" href="#t2154">2154</a></span><span class="t">                        <span class="nam">top_p</span><span class="op">=</span><span class="nam">top_p</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2155" href="#t2155">2155</a></span><span class="t">                        <span class="nam">temperature</span><span class="op">=</span><span class="nam">temperature</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2156" href="#t2156">2156</a></span><span class="t">                        <span class="nam">freq_penalty</span><span class="op">=</span><span class="nam">freq_penalty</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2157" href="#t2157">2157</a></span><span class="t">                        <span class="nam">tokens</span><span class="op">=</span><span class="nam">tokens</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2158" href="#t2158">2158</a></span><span class="t">                    <span class="op">)</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">devices</span><span class="op">.</span><span class="nam">get_device_for_block_index</span><span class="op">(</span><span class="num">0</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2159" href="#t2159">2159</a></span><span class="t">                <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2160" href="#t2160">2160</a></span><span class="t">                    <span class="nam">sampled_tokens</span> <span class="op">=</span> <span class="nam">final_logits</span><span class="op">.</span><span class="nam">argmax</span><span class="op">(</span><span class="op">-</span><span class="num">1</span><span class="op">)</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2161" href="#t2161">2161</a></span><span class="t">                        <span class="nam">devices</span><span class="op">.</span><span class="nam">get_device_for_block_index</span><span class="op">(</span><span class="num">0</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2162" href="#t2162">2162</a></span><span class="t">                    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2163" href="#t2163">2163</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2164" href="#t2164">2164</a></span><span class="t">                <span class="key">if</span> <span class="nam">stop_at_eos</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2165" href="#t2165">2165</a></span><span class="t">                    <span class="com"># For all unfinished sequences, add on the next token. If a sequence was</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2166" href="#t2166">2166</a></span><span class="t">                    <span class="com"># finished, throw away the generated token and add eos_token_for_padding</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2167" href="#t2167">2167</a></span><span class="t">                    <span class="com"># instead.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2168" href="#t2168">2168</a></span><span class="t">                    <span class="nam">sampled_tokens</span><span class="op">[</span><span class="nam">finished_sequences</span><span class="op">]</span> <span class="op">=</span> <span class="nam">eos_token_for_padding</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2169" href="#t2169">2169</a></span><span class="t">                    <span class="nam">finished_sequences</span><span class="op">.</span><span class="nam">logical_or_</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2170" href="#t2170">2170</a></span><span class="t">                        <span class="nam">torch</span><span class="op">.</span><span class="nam">isin</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2171" href="#t2171">2171</a></span><span class="t">                            <span class="nam">sampled_tokens</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">device</span><span class="op">)</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2172" href="#t2172">2172</a></span><span class="t">                            <span class="nam">torch</span><span class="op">.</span><span class="nam">tensor</span><span class="op">(</span><span class="nam">stop_tokens</span><span class="op">)</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">device</span><span class="op">)</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2173" href="#t2173">2173</a></span><span class="t">                        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2174" href="#t2174">2174</a></span><span class="t">                    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2175" href="#t2175">2175</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2176" href="#t2176">2176</a></span><span class="t">                <span class="nam">tokens</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">cat</span><span class="op">(</span><span class="op">[</span><span class="nam">tokens</span><span class="op">,</span> <span class="nam">sampled_tokens</span><span class="op">.</span><span class="nam">unsqueeze</span><span class="op">(</span><span class="op">-</span><span class="num">1</span><span class="op">)</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="op">-</span><span class="num">1</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t2177" href="#t2177">2177</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2178" href="#t2178">2178</a></span><span class="t">            <span class="key">if</span> <span class="nam">return_type</span> <span class="op">==</span> <span class="str">"str"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2179" href="#t2179">2179</a></span><span class="t">                <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">default_prepend_bos</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2180" href="#t2180">2180</a></span><span class="t">                    <span class="com"># If we prepended a BOS token, remove it when returning output.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2181" href="#t2181">2181</a></span><span class="t">                    <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">decode</span><span class="op">(</span><span class="nam">tokens</span><span class="op">[</span><span class="num">0</span><span class="op">,</span> <span class="num">1</span><span class="op">:</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2182" href="#t2182">2182</a></span><span class="t">                <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2183" href="#t2183">2183</a></span><span class="t">                    <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">decode</span><span class="op">(</span><span class="nam">tokens</span><span class="op">[</span><span class="num">0</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2184" href="#t2184">2184</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2185" href="#t2185">2185</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2186" href="#t2186">2186</a></span><span class="t">                <span class="key">return</span> <span class="nam">tokens</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2178" href="#t2178">2178</a></span><span class="t">                <span class="key">if</span> <span class="nam">stop_at_eos</span> <span class="key">and</span> <span class="nam">finished_sequences</span><span class="op">.</span><span class="nam">all</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2179" href="#t2179">2179</a></span><span class="t">                    <span class="key">break</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2180" href="#t2180">2180</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2181" href="#t2181">2181</a></span><span class="t">            <span class="key">if</span> <span class="nam">return_type</span> <span class="op">==</span> <span class="str">"str"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2182" href="#t2182">2182</a></span><span class="t">                <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">default_prepend_bos</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2183" href="#t2183">2183</a></span><span class="t">                    <span class="com"># If we prepended a BOS token, remove it when returning output.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2184" href="#t2184">2184</a></span><span class="t">                    <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">decode</span><span class="op">(</span><span class="nam">tokens</span><span class="op">[</span><span class="num">0</span><span class="op">,</span> <span class="num">1</span><span class="op">:</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2185" href="#t2185">2185</a></span><span class="t">                <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2186" href="#t2186">2186</a></span><span class="t">                    <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">tokenizer</span><span class="op">.</span><span class="nam">decode</span><span class="op">(</span><span class="nam">tokens</span><span class="op">[</span><span class="num">0</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t2187" href="#t2187">2187</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2188" href="#t2188">2188</a></span><span class="t">    <span class="com"># Give access to all weights as properties.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2189" href="#t2189">2189</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2190" href="#t2190">2190</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_U</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"d_model d_vocab"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2191" href="#t2191">2191</a></span><span class="t">        <span class="str">"""Convenience to get the unembedding matrix.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2192" href="#t2192">2192</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2193" href="#t2193">2193</a></span><span class="t"><span class="str">        I.e. the linear map from the final residual stream to the output logits).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2194" href="#t2194">2194</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2195" href="#t2195">2195</a></span><span class="t">        <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">unembed</span><span class="op">.</span><span class="nam">W_U</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2196" href="#t2196">2196</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2197" href="#t2197">2197</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2198" href="#t2198">2198</a></span><span class="t">    <span class="key">def</span> <span class="nam">b_U</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"d_vocab"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2199" href="#t2199">2199</a></span><span class="t">        <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">unembed</span><span class="op">.</span><span class="nam">b_U</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2200" href="#t2200">2200</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2201" href="#t2201">2201</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2202" href="#t2202">2202</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_E</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"d_vocab d_model"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2203" href="#t2203">2203</a></span><span class="t">        <span class="str">"""Convenience to get the embedding matrix."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2204" href="#t2204">2204</a></span><span class="t">        <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">embed</span><span class="op">.</span><span class="nam">W_E</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2205" href="#t2205">2205</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2206" href="#t2206">2206</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2207" href="#t2207">2207</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_pos</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_ctx d_model"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2208" href="#t2208">2208</a></span><span class="t">        <span class="str">"""Convenience function to get the positional embedding.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2209" href="#t2209">2209</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2210" href="#t2210">2210</a></span><span class="t"><span class="str">        Only works on models with absolute positional embeddings!</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2211" href="#t2211">2211</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2212" href="#t2212">2212</a></span><span class="t">        <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">pos_embed</span><span class="op">.</span><span class="nam">W_pos</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2213" href="#t2213">2213</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2214" href="#t2214">2214</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2215" href="#t2215">2215</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_E_pos</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"d_vocab+n_ctx d_model"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2216" href="#t2216">2216</a></span><span class="t">        <span class="str">"""Concatenated W_E and W_pos.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2217" href="#t2217">2217</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2218" href="#t2218">2218</a></span><span class="t"><span class="str">        Used as a full (overcomplete) basis of the input space, useful for full QK and full OV</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2219" href="#t2219">2219</a></span><span class="t"><span class="str">        circuits.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2220" href="#t2220">2220</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2221" href="#t2221">2221</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">cat</span><span class="op">(</span><span class="op">[</span><span class="nam">self</span><span class="op">.</span><span class="nam">W_E</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">W_pos</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2222" href="#t2222">2222</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2223" href="#t2223">2223</a></span><span class="t">    <span class="com"># Layer-specific weights are stacked into one massive tensor and given as properties for</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2224" href="#t2224">2224</a></span><span class="t">    <span class="com"># convenience and a cache is used to avoid repeated computation. Often a useful convenience when</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2225" href="#t2225">2225</a></span><span class="t">    <span class="com"># we want to do analysis on weights across all layers. If GPU memory is a bottleneck, don't use</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2226" href="#t2226">2226</a></span><span class="t">    <span class="com"># these properties!</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2227" href="#t2227">2227</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2228" href="#t2228">2228</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2229" href="#t2229">2229</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_K</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers n_heads d_model d_head"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2230" href="#t2230">2230</a></span><span class="t">        <span class="str">"""Stack the key weights across all layers."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2231" href="#t2231">2231</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">W_K</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2231&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2231&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2231 didn't run the list comprehension on line 2231, 2) line 2231 didn't return from function 'W_K', because the return on line 2231 wasn't executed</span></span></p>
-    <p class="pln"><span class="n"><a id="t2232" href="#t2232">2232</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2233" href="#t2233">2233</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2234" href="#t2234">2234</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_Q</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers n_heads d_model d_head"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2235" href="#t2235">2235</a></span><span class="t">        <span class="str">"""Stack the query weights across all layers."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2236" href="#t2236">2236</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">W_Q</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2236&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2236&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2236 didn't run the list comprehension on line 2236, 2) line 2236 didn't return from function 'W_Q', because the return on line 2236 wasn't executed</span></span></p>
-    <p class="pln"><span class="n"><a id="t2237" href="#t2237">2237</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2238" href="#t2238">2238</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2239" href="#t2239">2239</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_V</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers n_heads d_model d_head"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2240" href="#t2240">2240</a></span><span class="t">        <span class="str">"""Stack the value weights across all layers."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2241" href="#t2241">2241</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">W_V</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2241&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2241&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2241 didn't run the list comprehension on line 2241, 2) line 2241 didn't return from function 'W_V', because the return on line 2241 wasn't executed</span></span></p>
-    <p class="pln"><span class="n"><a id="t2242" href="#t2242">2242</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2243" href="#t2243">2243</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2244" href="#t2244">2244</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_O</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers n_heads d_head d_model"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2245" href="#t2245">2245</a></span><span class="t">        <span class="str">"""Stack the attn output weights across all layers."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2246" href="#t2246">2246</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">W_O</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2246&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2246&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2246 didn't run the list comprehension on line 2246, 2) line 2246 didn't return from function 'W_O', because the return on line 2246 wasn't executed</span></span></p>
-    <p class="pln"><span class="n"><a id="t2247" href="#t2247">2247</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2248" href="#t2248">2248</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2249" href="#t2249">2249</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_in</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers d_model d_mlp"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2250" href="#t2250">2250</a></span><span class="t">        <span class="str">"""Stack the MLP input weights across all layers."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2251" href="#t2251">2251</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">mlp</span><span class="op">.</span><span class="nam">W_in</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2251&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2251&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2251 didn't run the list comprehension on line 2251, 2) line 2251 didn't return from function 'W_in', because the return on line 2251 wasn't executed</span></span></p>
-    <p class="pln"><span class="n"><a id="t2252" href="#t2252">2252</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2253" href="#t2253">2253</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2254" href="#t2254">2254</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_gate</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Union</span><span class="op">[</span><span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers d_model d_mlp"</span><span class="op">]</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2255" href="#t2255">2255</a></span><span class="t">        <span class="str">"""Stack the MLP gate weights across all layers.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2256" href="#t2256">2256</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2257" href="#t2257">2257</a></span><span class="t"><span class="str">        Only works for models with gated MLPs.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2258" href="#t2258">2258</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2259" href="#t2259">2259</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">gated_mlp</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2260" href="#t2260">2260</a></span><span class="t">            <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">mlp</span><span class="op">.</span><span class="nam">W_gate</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2261" href="#t2261">2261</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2262" href="#t2262">2262</a></span><span class="t">            <span class="key">return</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2263" href="#t2263">2263</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2264" href="#t2264">2264</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2265" href="#t2265">2265</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_out</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers d_mlp d_model"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2266" href="#t2266">2266</a></span><span class="t">        <span class="str">"""Stack the MLP output weights across all layers."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2267" href="#t2267">2267</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">mlp</span><span class="op">.</span><span class="nam">W_out</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2267&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2267&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2267 didn't run the list comprehension on line 2267, 2) line 2267 didn't return from function 'W_out', because the return on line 2267 wasn't executed</span></span></p>
-    <p class="pln"><span class="n"><a id="t2268" href="#t2268">2268</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2269" href="#t2269">2269</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2270" href="#t2270">2270</a></span><span class="t">    <span class="key">def</span> <span class="nam">b_K</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers n_heads d_head"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2271" href="#t2271">2271</a></span><span class="t">        <span class="str">"""Stack the key biases across all layers."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2272" href="#t2272">2272</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">b_K</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2272&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2272&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2272 didn't run the list comprehension on line 2272, 2) line 2272 didn't return from function 'b_K', because the return on line 2272 wasn't executed</span></span></p>
-    <p class="pln"><span class="n"><a id="t2273" href="#t2273">2273</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2274" href="#t2274">2274</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2275" href="#t2275">2275</a></span><span class="t">    <span class="key">def</span> <span class="nam">b_Q</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers n_heads d_head"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2276" href="#t2276">2276</a></span><span class="t">        <span class="str">"""Stack the query biases across all layers."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2277" href="#t2277">2277</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">b_Q</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2277&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2277&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2277 didn't run the list comprehension on line 2277, 2) line 2277 didn't return from function 'b_Q', because the return on line 2277 wasn't executed</span></span></p>
-    <p class="pln"><span class="n"><a id="t2278" href="#t2278">2278</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2279" href="#t2279">2279</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2280" href="#t2280">2280</a></span><span class="t">    <span class="key">def</span> <span class="nam">b_V</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers n_heads d_head"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2281" href="#t2281">2281</a></span><span class="t">        <span class="str">"""Stack the value biases across all layers."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2282" href="#t2282">2282</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">b_V</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2282&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2282&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2282 didn't run the list comprehension on line 2282, 2) line 2282 didn't return from function 'b_V', because the return on line 2282 wasn't executed</span></span></p>
-    <p class="pln"><span class="n"><a id="t2283" href="#t2283">2283</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2284" href="#t2284">2284</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2285" href="#t2285">2285</a></span><span class="t">    <span class="key">def</span> <span class="nam">b_O</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers d_model"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2286" href="#t2286">2286</a></span><span class="t">        <span class="str">"""Stack the attn output biases across all layers."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2287" href="#t2287">2287</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">b_O</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2287&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2287&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2287 didn't run the list comprehension on line 2287, 2) line 2287 didn't return from function 'b_O', because the return on line 2287 wasn't executed</span></span></p>
-    <p class="pln"><span class="n"><a id="t2288" href="#t2288">2288</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2289" href="#t2289">2289</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2290" href="#t2290">2290</a></span><span class="t">    <span class="key">def</span> <span class="nam">b_in</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers d_mlp"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2291" href="#t2291">2291</a></span><span class="t">        <span class="str">"""Stack the MLP input biases across all layers."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2292" href="#t2292">2292</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">mlp</span><span class="op">.</span><span class="nam">b_in</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2292&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2292&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2292 didn't run the list comprehension on line 2292, 2) line 2292 didn't return from function 'b_in', because the return on line 2292 wasn't executed</span></span></p>
-    <p class="pln"><span class="n"><a id="t2293" href="#t2293">2293</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2294" href="#t2294">2294</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2295" href="#t2295">2295</a></span><span class="t">    <span class="key">def</span> <span class="nam">b_out</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers d_model"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2296" href="#t2296">2296</a></span><span class="t">        <span class="str">"""Stack the MLP output biases across all layers."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2297" href="#t2297">2297</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">mlp</span><span class="op">.</span><span class="nam">b_out</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2297&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2297&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2297 didn't run the list comprehension on line 2297, 2) line 2297 didn't return from function 'b_out', because the return on line 2297 wasn't executed</span></span></p>
-    <p class="pln"><span class="n"><a id="t2298" href="#t2298">2298</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2299" href="#t2299">2299</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2300" href="#t2300">2300</a></span><span class="t">    <span class="key">def</span> <span class="nam">QK</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2301" href="#t2301">2301</a></span><span class="t">        <span class="key">return</span> <span class="nam">FactoredMatrix</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">W_Q</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">W_K</span><span class="op">.</span><span class="nam">transpose</span><span class="op">(</span><span class="op">-</span><span class="num">2</span><span class="op">,</span> <span class="op">-</span><span class="num">1</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2302" href="#t2302">2302</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2303" href="#t2303">2303</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2304" href="#t2304">2304</a></span><span class="t">    <span class="key">def</span> <span class="nam">OV</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2305" href="#t2305">2305</a></span><span class="t">        <span class="key">return</span> <span class="nam">FactoredMatrix</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">W_V</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">W_O</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2306" href="#t2306">2306</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2307" href="#t2307">2307</a></span><span class="t">    <span class="com"># Various utility functions</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2308" href="#t2308">2308</a></span><span class="t">    <span class="key">def</span> <span class="nam">accumulated_bias</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2309" href="#t2309">2309</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span> <span class="nam">layer</span><span class="op">:</span> <span class="nam">int</span><span class="op">,</span> <span class="nam">mlp_input</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span> <span class="nam">include_mlp_biases</span><span class="op">=</span><span class="key">True</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2310" href="#t2310">2310</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"d_model"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2311" href="#t2311">2311</a></span><span class="t">        <span class="str">"""Accumulated Bias.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2312" href="#t2312">2312</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2313" href="#t2313">2313</a></span><span class="t"><span class="str">        Returns the accumulated bias from all layer outputs (ie the b_Os and b_outs), up to the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2314" href="#t2314">2314</a></span><span class="t"><span class="str">        input of layer L.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2188" href="#t2188">2188</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2189" href="#t2189">2189</a></span><span class="t">                <span class="key">return</span> <span class="nam">tokens</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2190" href="#t2190">2190</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2191" href="#t2191">2191</a></span><span class="t">    <span class="com"># Give access to all weights as properties.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2192" href="#t2192">2192</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2193" href="#t2193">2193</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_U</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"d_model d_vocab"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2194" href="#t2194">2194</a></span><span class="t">        <span class="str">"""Convenience to get the unembedding matrix.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2195" href="#t2195">2195</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2196" href="#t2196">2196</a></span><span class="t"><span class="str">        I.e. the linear map from the final residual stream to the output logits).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2197" href="#t2197">2197</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2198" href="#t2198">2198</a></span><span class="t">        <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">unembed</span><span class="op">.</span><span class="nam">W_U</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2199" href="#t2199">2199</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2200" href="#t2200">2200</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2201" href="#t2201">2201</a></span><span class="t">    <span class="key">def</span> <span class="nam">b_U</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"d_vocab"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2202" href="#t2202">2202</a></span><span class="t">        <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">unembed</span><span class="op">.</span><span class="nam">b_U</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2203" href="#t2203">2203</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2204" href="#t2204">2204</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2205" href="#t2205">2205</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_E</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"d_vocab d_model"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2206" href="#t2206">2206</a></span><span class="t">        <span class="str">"""Convenience to get the embedding matrix."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2207" href="#t2207">2207</a></span><span class="t">        <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">embed</span><span class="op">.</span><span class="nam">W_E</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2208" href="#t2208">2208</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2209" href="#t2209">2209</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2210" href="#t2210">2210</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_pos</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_ctx d_model"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2211" href="#t2211">2211</a></span><span class="t">        <span class="str">"""Convenience function to get the positional embedding.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2212" href="#t2212">2212</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2213" href="#t2213">2213</a></span><span class="t"><span class="str">        Only works on models with absolute positional embeddings!</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2214" href="#t2214">2214</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2215" href="#t2215">2215</a></span><span class="t">        <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">pos_embed</span><span class="op">.</span><span class="nam">W_pos</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2216" href="#t2216">2216</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2217" href="#t2217">2217</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2218" href="#t2218">2218</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_E_pos</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"d_vocab+n_ctx d_model"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2219" href="#t2219">2219</a></span><span class="t">        <span class="str">"""Concatenated W_E and W_pos.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2220" href="#t2220">2220</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2221" href="#t2221">2221</a></span><span class="t"><span class="str">        Used as a full (overcomplete) basis of the input space, useful for full QK and full OV</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2222" href="#t2222">2222</a></span><span class="t"><span class="str">        circuits.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2223" href="#t2223">2223</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2224" href="#t2224">2224</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">cat</span><span class="op">(</span><span class="op">[</span><span class="nam">self</span><span class="op">.</span><span class="nam">W_E</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">W_pos</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2225" href="#t2225">2225</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2226" href="#t2226">2226</a></span><span class="t">    <span class="com"># Layer-specific weights are stacked into one massive tensor and given as properties for</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2227" href="#t2227">2227</a></span><span class="t">    <span class="com"># convenience and a cache is used to avoid repeated computation. Often a useful convenience when</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2228" href="#t2228">2228</a></span><span class="t">    <span class="com"># we want to do analysis on weights across all layers. If GPU memory is a bottleneck, don't use</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2229" href="#t2229">2229</a></span><span class="t">    <span class="com"># these properties!</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2230" href="#t2230">2230</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2231" href="#t2231">2231</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2232" href="#t2232">2232</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_K</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers n_heads d_model d_head"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2233" href="#t2233">2233</a></span><span class="t">        <span class="str">"""Stack the key weights across all layers."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2234" href="#t2234">2234</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">W_K</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2234&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2234&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2234 didn't run the list comprehension on line 2234, 2) line 2234 didn't return from function 'W_K', because the return on line 2234 wasn't executed</span></span></p>
+    <p class="pln"><span class="n"><a id="t2235" href="#t2235">2235</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2236" href="#t2236">2236</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2237" href="#t2237">2237</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_Q</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers n_heads d_model d_head"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2238" href="#t2238">2238</a></span><span class="t">        <span class="str">"""Stack the query weights across all layers."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2239" href="#t2239">2239</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">W_Q</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2239&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2239&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2239 didn't run the list comprehension on line 2239, 2) line 2239 didn't return from function 'W_Q', because the return on line 2239 wasn't executed</span></span></p>
+    <p class="pln"><span class="n"><a id="t2240" href="#t2240">2240</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2241" href="#t2241">2241</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2242" href="#t2242">2242</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_V</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers n_heads d_model d_head"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2243" href="#t2243">2243</a></span><span class="t">        <span class="str">"""Stack the value weights across all layers."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2244" href="#t2244">2244</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">W_V</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2244&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2244&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2244 didn't run the list comprehension on line 2244, 2) line 2244 didn't return from function 'W_V', because the return on line 2244 wasn't executed</span></span></p>
+    <p class="pln"><span class="n"><a id="t2245" href="#t2245">2245</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2246" href="#t2246">2246</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2247" href="#t2247">2247</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_O</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers n_heads d_head d_model"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2248" href="#t2248">2248</a></span><span class="t">        <span class="str">"""Stack the attn output weights across all layers."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2249" href="#t2249">2249</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">W_O</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2249&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2249&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2249 didn't run the list comprehension on line 2249, 2) line 2249 didn't return from function 'W_O', because the return on line 2249 wasn't executed</span></span></p>
+    <p class="pln"><span class="n"><a id="t2250" href="#t2250">2250</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2251" href="#t2251">2251</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2252" href="#t2252">2252</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_in</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers d_model d_mlp"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2253" href="#t2253">2253</a></span><span class="t">        <span class="str">"""Stack the MLP input weights across all layers."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2254" href="#t2254">2254</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">mlp</span><span class="op">.</span><span class="nam">W_in</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2254&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2254&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2254 didn't run the list comprehension on line 2254, 2) line 2254 didn't return from function 'W_in', because the return on line 2254 wasn't executed</span></span></p>
+    <p class="pln"><span class="n"><a id="t2255" href="#t2255">2255</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2256" href="#t2256">2256</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2257" href="#t2257">2257</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_gate</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Union</span><span class="op">[</span><span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers d_model d_mlp"</span><span class="op">]</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2258" href="#t2258">2258</a></span><span class="t">        <span class="str">"""Stack the MLP gate weights across all layers.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2259" href="#t2259">2259</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2260" href="#t2260">2260</a></span><span class="t"><span class="str">        Only works for models with gated MLPs.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2261" href="#t2261">2261</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2262" href="#t2262">2262</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">gated_mlp</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2263" href="#t2263">2263</a></span><span class="t">            <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">mlp</span><span class="op">.</span><span class="nam">W_gate</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2264" href="#t2264">2264</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2265" href="#t2265">2265</a></span><span class="t">            <span class="key">return</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2266" href="#t2266">2266</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2267" href="#t2267">2267</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2268" href="#t2268">2268</a></span><span class="t">    <span class="key">def</span> <span class="nam">W_out</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers d_mlp d_model"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2269" href="#t2269">2269</a></span><span class="t">        <span class="str">"""Stack the MLP output weights across all layers."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2270" href="#t2270">2270</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">mlp</span><span class="op">.</span><span class="nam">W_out</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2270&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2270&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2270 didn't run the list comprehension on line 2270, 2) line 2270 didn't return from function 'W_out', because the return on line 2270 wasn't executed</span></span></p>
+    <p class="pln"><span class="n"><a id="t2271" href="#t2271">2271</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2272" href="#t2272">2272</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2273" href="#t2273">2273</a></span><span class="t">    <span class="key">def</span> <span class="nam">b_K</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers n_heads d_head"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2274" href="#t2274">2274</a></span><span class="t">        <span class="str">"""Stack the key biases across all layers."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2275" href="#t2275">2275</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">b_K</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2275&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2275&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2275 didn't run the list comprehension on line 2275, 2) line 2275 didn't return from function 'b_K', because the return on line 2275 wasn't executed</span></span></p>
+    <p class="pln"><span class="n"><a id="t2276" href="#t2276">2276</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2277" href="#t2277">2277</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2278" href="#t2278">2278</a></span><span class="t">    <span class="key">def</span> <span class="nam">b_Q</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers n_heads d_head"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2279" href="#t2279">2279</a></span><span class="t">        <span class="str">"""Stack the query biases across all layers."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2280" href="#t2280">2280</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">b_Q</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2280&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2280&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2280 didn't run the list comprehension on line 2280, 2) line 2280 didn't return from function 'b_Q', because the return on line 2280 wasn't executed</span></span></p>
+    <p class="pln"><span class="n"><a id="t2281" href="#t2281">2281</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2282" href="#t2282">2282</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2283" href="#t2283">2283</a></span><span class="t">    <span class="key">def</span> <span class="nam">b_V</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers n_heads d_head"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2284" href="#t2284">2284</a></span><span class="t">        <span class="str">"""Stack the value biases across all layers."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2285" href="#t2285">2285</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">b_V</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2285&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2285&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2285 didn't run the list comprehension on line 2285, 2) line 2285 didn't return from function 'b_V', because the return on line 2285 wasn't executed</span></span></p>
+    <p class="pln"><span class="n"><a id="t2286" href="#t2286">2286</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2287" href="#t2287">2287</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2288" href="#t2288">2288</a></span><span class="t">    <span class="key">def</span> <span class="nam">b_O</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers d_model"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2289" href="#t2289">2289</a></span><span class="t">        <span class="str">"""Stack the attn output biases across all layers."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2290" href="#t2290">2290</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">b_O</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2290&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2290&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2290 didn't run the list comprehension on line 2290, 2) line 2290 didn't return from function 'b_O', because the return on line 2290 wasn't executed</span></span></p>
+    <p class="pln"><span class="n"><a id="t2291" href="#t2291">2291</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2292" href="#t2292">2292</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2293" href="#t2293">2293</a></span><span class="t">    <span class="key">def</span> <span class="nam">b_in</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers d_mlp"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2294" href="#t2294">2294</a></span><span class="t">        <span class="str">"""Stack the MLP input biases across all layers."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2295" href="#t2295">2295</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">mlp</span><span class="op">.</span><span class="nam">b_in</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2295&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2295&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2295 didn't run the list comprehension on line 2295, 2) line 2295 didn't return from function 'b_in', because the return on line 2295 wasn't executed</span></span></p>
+    <p class="pln"><span class="n"><a id="t2296" href="#t2296">2296</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2297" href="#t2297">2297</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2298" href="#t2298">2298</a></span><span class="t">    <span class="key">def</span> <span class="nam">b_out</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers d_model"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2299" href="#t2299">2299</a></span><span class="t">        <span class="str">"""Stack the MLP output biases across all layers."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2300" href="#t2300">2300</a></span><span class="t">        <span class="key">return</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">stack</span><span class="op">(</span><span class="op">[</span><span class="nam">block</span><span class="op">.</span><span class="nam">mlp</span><span class="op">.</span><span class="nam">b_out</span> <span class="key">for</span> <span class="nam">block</span> <span class="key">in</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">]</span><span class="op">,</span> <span class="nam">dim</span><span class="op">=</span><span class="num">0</span><span class="op">)</span>&nbsp;</span><span class="r"><span class="annotate short">2300&#x202F;&#x219B;&#x202F;exit,&nbsp;&nbsp; 2300&#x202F;&#x219B;&#x202F;exit</span><span class="annotate long">2 missed branches: 1) line 2300 didn't run the list comprehension on line 2300, 2) line 2300 didn't return from function 'b_out', because the return on line 2300 wasn't executed</span></span></p>
+    <p class="pln"><span class="n"><a id="t2301" href="#t2301">2301</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2302" href="#t2302">2302</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2303" href="#t2303">2303</a></span><span class="t">    <span class="key">def</span> <span class="nam">QK</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2304" href="#t2304">2304</a></span><span class="t">        <span class="key">return</span> <span class="nam">FactoredMatrix</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">W_Q</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">W_K</span><span class="op">.</span><span class="nam">transpose</span><span class="op">(</span><span class="op">-</span><span class="num">2</span><span class="op">,</span> <span class="op">-</span><span class="num">1</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2305" href="#t2305">2305</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2306" href="#t2306">2306</a></span><span class="t">    <span class="op">@</span><span class="nam">property</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2307" href="#t2307">2307</a></span><span class="t">    <span class="key">def</span> <span class="nam">OV</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2308" href="#t2308">2308</a></span><span class="t">        <span class="key">return</span> <span class="nam">FactoredMatrix</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">W_V</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">W_O</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2309" href="#t2309">2309</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2310" href="#t2310">2310</a></span><span class="t">    <span class="com"># Various utility functions</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2311" href="#t2311">2311</a></span><span class="t">    <span class="key">def</span> <span class="nam">accumulated_bias</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2312" href="#t2312">2312</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span> <span class="nam">layer</span><span class="op">:</span> <span class="nam">int</span><span class="op">,</span> <span class="nam">mlp_input</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span> <span class="nam">include_mlp_biases</span><span class="op">=</span><span class="key">True</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2313" href="#t2313">2313</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"d_model"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2314" href="#t2314">2314</a></span><span class="t">        <span class="str">"""Accumulated Bias.</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t2315" href="#t2315">2315</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2316" href="#t2316">2316</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2317" href="#t2317">2317</a></span><span class="t"><span class="str">            layer (int): Layer number, in [0, n_layers]. layer==0 means no layers, layer==n_layers</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2318" href="#t2318">2318</a></span><span class="t"><span class="str">                means all layers.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2319" href="#t2319">2319</a></span><span class="t"><span class="str">            mlp_input (bool): If True, we take the bias up to the input of the MLP</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2320" href="#t2320">2320</a></span><span class="t"><span class="str">                of layer L (ie we include the bias from the attention output of the current layer,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2321" href="#t2321">2321</a></span><span class="t"><span class="str">                otherwise just biases from previous layers)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2322" href="#t2322">2322</a></span><span class="t"><span class="str">            include_mlp_biases (bool): Whether to include the biases of MLP layers. Often useful to</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2323" href="#t2323">2323</a></span><span class="t"><span class="str">                have as False if we're expanding attn_out into individual heads, but keeping mlp_out</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2324" href="#t2324">2324</a></span><span class="t"><span class="str">                as is.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2325" href="#t2325">2325</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2326" href="#t2326">2326</a></span><span class="t"><span class="str">        Returns:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2327" href="#t2327">2327</a></span><span class="t"><span class="str">            bias (torch.Tensor): [d_model], accumulated bias</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2328" href="#t2328">2328</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2329" href="#t2329">2329</a></span><span class="t">        <span class="nam">accumulated_bias</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">zeros</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">d_model</span><span class="op">,</span> <span class="nam">device</span><span class="op">=</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">device</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2330" href="#t2330">2330</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2331" href="#t2331">2331</a></span><span class="t">        <span class="key">for</span> <span class="nam">i</span> <span class="key">in</span> <span class="nam">range</span><span class="op">(</span><span class="nam">layer</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2332" href="#t2332">2332</a></span><span class="t">            <span class="nam">accumulated_bias</span> <span class="op">+=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">[</span><span class="nam">i</span><span class="op">]</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">b_O</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2333" href="#t2333">2333</a></span><span class="t">            <span class="key">if</span> <span class="nam">include_mlp_biases</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2334" href="#t2334">2334</a></span><span class="t">                <span class="nam">accumulated_bias</span> <span class="op">+=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">[</span><span class="nam">i</span><span class="op">]</span><span class="op">.</span><span class="nam">mlp</span><span class="op">.</span><span class="nam">b_out</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t2335" href="#t2335">2335</a></span><span class="t">        <span class="key">if</span> <span class="nam">mlp_input</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2335&#x202F;&#x219B;&#x202F;2336</span><span class="annotate long">line 2335 didn't jump to line 2336, because the condition on line 2335 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2336" href="#t2336">2336</a></span><span class="t">            <span class="key">assert</span> <span class="nam">layer</span> <span class="op">&lt;</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span><span class="op">,</span> <span class="str">"Cannot include attn_bias from beyond the final layer"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2337" href="#t2337">2337</a></span><span class="t">            <span class="nam">accumulated_bias</span> <span class="op">+=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">[</span><span class="nam">layer</span><span class="op">]</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">b_O</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2338" href="#t2338">2338</a></span><span class="t">        <span class="key">return</span> <span class="nam">accumulated_bias</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2339" href="#t2339">2339</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2340" href="#t2340">2340</a></span><span class="t">    <span class="key">def</span> <span class="nam">all_composition_scores</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2341" href="#t2341">2341</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span> <span class="nam">mode</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2342" href="#t2342">2342</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers n_heads n_layers n_heads"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2343" href="#t2343">2343</a></span><span class="t">        <span class="str">"""All Composition Scores.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2344" href="#t2344">2344</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2345" href="#t2345">2345</a></span><span class="t"><span class="str">        Returns the Composition scores for all pairs of heads, as a L1, H1, L2, H2 tensor (which is</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2346" href="#t2346">2346</a></span><span class="t"><span class="str">        upper triangular on the first and third axes).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2316" href="#t2316">2316</a></span><span class="t"><span class="str">        Returns the accumulated bias from all layer outputs (ie the b_Os and b_outs), up to the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2317" href="#t2317">2317</a></span><span class="t"><span class="str">        input of layer L.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2318" href="#t2318">2318</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2319" href="#t2319">2319</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2320" href="#t2320">2320</a></span><span class="t"><span class="str">            layer (int): Layer number, in [0, n_layers]. layer==0 means no layers, layer==n_layers</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2321" href="#t2321">2321</a></span><span class="t"><span class="str">                means all layers.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2322" href="#t2322">2322</a></span><span class="t"><span class="str">            mlp_input (bool): If True, we take the bias up to the input of the MLP</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2323" href="#t2323">2323</a></span><span class="t"><span class="str">                of layer L (ie we include the bias from the attention output of the current layer,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2324" href="#t2324">2324</a></span><span class="t"><span class="str">                otherwise just biases from previous layers)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2325" href="#t2325">2325</a></span><span class="t"><span class="str">            include_mlp_biases (bool): Whether to include the biases of MLP layers. Often useful to</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2326" href="#t2326">2326</a></span><span class="t"><span class="str">                have as False if we're expanding attn_out into individual heads, but keeping mlp_out</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2327" href="#t2327">2327</a></span><span class="t"><span class="str">                as is.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2328" href="#t2328">2328</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2329" href="#t2329">2329</a></span><span class="t"><span class="str">        Returns:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2330" href="#t2330">2330</a></span><span class="t"><span class="str">            bias (torch.Tensor): [d_model], accumulated bias</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2331" href="#t2331">2331</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2332" href="#t2332">2332</a></span><span class="t">        <span class="nam">accumulated_bias</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">zeros</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">d_model</span><span class="op">,</span> <span class="nam">device</span><span class="op">=</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">device</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2333" href="#t2333">2333</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2334" href="#t2334">2334</a></span><span class="t">        <span class="key">for</span> <span class="nam">i</span> <span class="key">in</span> <span class="nam">range</span><span class="op">(</span><span class="nam">layer</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2335" href="#t2335">2335</a></span><span class="t">            <span class="nam">accumulated_bias</span> <span class="op">+=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">[</span><span class="nam">i</span><span class="op">]</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">b_O</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2336" href="#t2336">2336</a></span><span class="t">            <span class="key">if</span> <span class="nam">include_mlp_biases</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2337" href="#t2337">2337</a></span><span class="t">                <span class="nam">accumulated_bias</span> <span class="op">+=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">[</span><span class="nam">i</span><span class="op">]</span><span class="op">.</span><span class="nam">mlp</span><span class="op">.</span><span class="nam">b_out</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t2338" href="#t2338">2338</a></span><span class="t">        <span class="key">if</span> <span class="nam">mlp_input</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">2338&#x202F;&#x219B;&#x202F;2339</span><span class="annotate long">line 2338 didn't jump to line 2339, because the condition on line 2338 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2339" href="#t2339">2339</a></span><span class="t">            <span class="key">assert</span> <span class="nam">layer</span> <span class="op">&lt;</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span><span class="op">,</span> <span class="str">"Cannot include attn_bias from beyond the final layer"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2340" href="#t2340">2340</a></span><span class="t">            <span class="nam">accumulated_bias</span> <span class="op">+=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">blocks</span><span class="op">[</span><span class="nam">layer</span><span class="op">]</span><span class="op">.</span><span class="nam">attn</span><span class="op">.</span><span class="nam">b_O</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2341" href="#t2341">2341</a></span><span class="t">        <span class="key">return</span> <span class="nam">accumulated_bias</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2342" href="#t2342">2342</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2343" href="#t2343">2343</a></span><span class="t">    <span class="key">def</span> <span class="nam">all_composition_scores</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2344" href="#t2344">2344</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span> <span class="nam">mode</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2345" href="#t2345">2345</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"n_layers n_heads n_layers n_heads"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2346" href="#t2346">2346</a></span><span class="t">        <span class="str">"""All Composition Scores.</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t2347" href="#t2347">2347</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2348" href="#t2348">2348</a></span><span class="t"><span class="str">        See</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2349" href="#t2349">2349</a></span><span class="t"><span class="str">        https://transformer-circuits.pub/2021/framework/index.html#:~:text=The%20above%20diagram%20shows%20Q%2D%2C%20K%2D%2C%20and%20V%2DComposition</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2350" href="#t2350">2350</a></span><span class="t"><span class="str">        for three metrics used.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2351" href="#t2351">2351</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2352" href="#t2352">2352</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2353" href="#t2353">2353</a></span><span class="t"><span class="str">            mode (str): One of ["Q", "K", "V"], the mode to use for the composition score.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2354" href="#t2354">2354</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2355" href="#t2355">2355</a></span><span class="t">        <span class="nam">left</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">OV</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2356" href="#t2356">2356</a></span><span class="t">        <span class="key">if</span> <span class="nam">mode</span> <span class="op">==</span> <span class="str">"Q"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2357" href="#t2357">2357</a></span><span class="t">            <span class="nam">right</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">QK</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2358" href="#t2358">2358</a></span><span class="t">        <span class="key">elif</span> <span class="nam">mode</span> <span class="op">==</span> <span class="str">"K"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2359" href="#t2359">2359</a></span><span class="t">            <span class="nam">right</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">QK</span><span class="op">.</span><span class="nam">T</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2360" href="#t2360">2360</a></span><span class="t">        <span class="key">elif</span> <span class="nam">mode</span> <span class="op">==</span> <span class="str">"V"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2361" href="#t2361">2361</a></span><span class="t">            <span class="nam">right</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">OV</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2362" href="#t2362">2362</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2363" href="#t2363">2363</a></span><span class="t">            <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span><span class="str">f"mode must be one of ['Q', 'K', 'V'] not {mode}"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2364" href="#t2364">2364</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2365" href="#t2365">2365</a></span><span class="t">        <span class="nam">scores</span> <span class="op">=</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">composition_scores</span><span class="op">(</span><span class="nam">left</span><span class="op">,</span> <span class="nam">right</span><span class="op">,</span> <span class="nam">broadcast_dims</span><span class="op">=</span><span class="key">True</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2366" href="#t2366">2366</a></span><span class="t">        <span class="com"># Mask scores to be zero for all pairs with the right head in the same layer or earlier</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2367" href="#t2367">2367</a></span><span class="t">        <span class="com"># layer than the left head.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2368" href="#t2368">2368</a></span><span class="t">        <span class="nam">mask</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2369" href="#t2369">2369</a></span><span class="t">            <span class="nam">torch</span><span class="op">.</span><span class="nam">arange</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span><span class="op">,</span> <span class="nam">device</span><span class="op">=</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">device</span><span class="op">)</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">,</span> <span class="key">None</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2370" href="#t2370">2370</a></span><span class="t">            <span class="op">&lt;</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">arange</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span><span class="op">,</span> <span class="nam">device</span><span class="op">=</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">device</span><span class="op">)</span><span class="op">[</span><span class="key">None</span><span class="op">,</span> <span class="key">None</span><span class="op">,</span> <span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2371" href="#t2371">2371</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2372" href="#t2372">2372</a></span><span class="t">        <span class="nam">scores</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">where</span><span class="op">(</span><span class="nam">mask</span><span class="op">,</span> <span class="nam">scores</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">zeros_like</span><span class="op">(</span><span class="nam">scores</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2373" href="#t2373">2373</a></span><span class="t">        <span class="key">return</span> <span class="nam">scores</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2374" href="#t2374">2374</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2375" href="#t2375">2375</a></span><span class="t">    <span class="key">def</span> <span class="nam">all_head_labels</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2376" href="#t2376">2376</a></span><span class="t">        <span class="str">"""Returns a list of all head names in the model."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2377" href="#t2377">2377</a></span><span class="t">        <span class="key">return</span> <span class="op">[</span><span class="str">f"L{l}H{h}"</span> <span class="key">for</span> <span class="nam">l</span> <span class="key">in</span> <span class="nam">range</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span><span class="op">)</span> <span class="key">for</span> <span class="nam">h</span> <span class="key">in</span> <span class="nam">range</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_heads</span><span class="op">)</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2378" href="#t2378">2378</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2379" href="#t2379">2379</a></span><span class="t">    <span class="key">def</span> <span class="nam">load_sample_training_dataset</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2380" href="#t2380">2380</a></span><span class="t">        <span class="str">"""Load Sample Training Dataset.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2348" href="#t2348">2348</a></span><span class="t"><span class="str">        Returns the Composition scores for all pairs of heads, as a L1, H1, L2, H2 tensor (which is</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2349" href="#t2349">2349</a></span><span class="t"><span class="str">        upper triangular on the first and third axes).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2350" href="#t2350">2350</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2351" href="#t2351">2351</a></span><span class="t"><span class="str">        See</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2352" href="#t2352">2352</a></span><span class="t"><span class="str">        https://transformer-circuits.pub/2021/framework/index.html#:~:text=The%20above%20diagram%20shows%20Q%2D%2C%20K%2D%2C%20and%20V%2DComposition</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2353" href="#t2353">2353</a></span><span class="t"><span class="str">        for three metrics used.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2354" href="#t2354">2354</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2355" href="#t2355">2355</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2356" href="#t2356">2356</a></span><span class="t"><span class="str">            mode (str): One of ["Q", "K", "V"], the mode to use for the composition score.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2357" href="#t2357">2357</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2358" href="#t2358">2358</a></span><span class="t">        <span class="nam">left</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">OV</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2359" href="#t2359">2359</a></span><span class="t">        <span class="key">if</span> <span class="nam">mode</span> <span class="op">==</span> <span class="str">"Q"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2360" href="#t2360">2360</a></span><span class="t">            <span class="nam">right</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">QK</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2361" href="#t2361">2361</a></span><span class="t">        <span class="key">elif</span> <span class="nam">mode</span> <span class="op">==</span> <span class="str">"K"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2362" href="#t2362">2362</a></span><span class="t">            <span class="nam">right</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">QK</span><span class="op">.</span><span class="nam">T</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2363" href="#t2363">2363</a></span><span class="t">        <span class="key">elif</span> <span class="nam">mode</span> <span class="op">==</span> <span class="str">"V"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2364" href="#t2364">2364</a></span><span class="t">            <span class="nam">right</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">OV</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2365" href="#t2365">2365</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2366" href="#t2366">2366</a></span><span class="t">            <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span><span class="str">f"mode must be one of ['Q', 'K', 'V'] not {mode}"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2367" href="#t2367">2367</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2368" href="#t2368">2368</a></span><span class="t">        <span class="nam">scores</span> <span class="op">=</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">composition_scores</span><span class="op">(</span><span class="nam">left</span><span class="op">,</span> <span class="nam">right</span><span class="op">,</span> <span class="nam">broadcast_dims</span><span class="op">=</span><span class="key">True</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2369" href="#t2369">2369</a></span><span class="t">        <span class="com"># Mask scores to be zero for all pairs with the right head in the same layer or earlier</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2370" href="#t2370">2370</a></span><span class="t">        <span class="com"># layer than the left head.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2371" href="#t2371">2371</a></span><span class="t">        <span class="nam">mask</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2372" href="#t2372">2372</a></span><span class="t">            <span class="nam">torch</span><span class="op">.</span><span class="nam">arange</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span><span class="op">,</span> <span class="nam">device</span><span class="op">=</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">device</span><span class="op">)</span><span class="op">[</span><span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">,</span> <span class="key">None</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2373" href="#t2373">2373</a></span><span class="t">            <span class="op">&lt;</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">arange</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span><span class="op">,</span> <span class="nam">device</span><span class="op">=</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">device</span><span class="op">)</span><span class="op">[</span><span class="key">None</span><span class="op">,</span> <span class="key">None</span><span class="op">,</span> <span class="op">:</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2374" href="#t2374">2374</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2375" href="#t2375">2375</a></span><span class="t">        <span class="nam">scores</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">where</span><span class="op">(</span><span class="nam">mask</span><span class="op">,</span> <span class="nam">scores</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">zeros_like</span><span class="op">(</span><span class="nam">scores</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2376" href="#t2376">2376</a></span><span class="t">        <span class="key">return</span> <span class="nam">scores</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2377" href="#t2377">2377</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2378" href="#t2378">2378</a></span><span class="t">    <span class="key">def</span> <span class="nam">all_head_labels</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2379" href="#t2379">2379</a></span><span class="t">        <span class="str">"""Returns a list of all head names in the model."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2380" href="#t2380">2380</a></span><span class="t">        <span class="key">return</span> <span class="op">[</span><span class="str">f"L{l}H{h}"</span> <span class="key">for</span> <span class="nam">l</span> <span class="key">in</span> <span class="nam">range</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_layers</span><span class="op">)</span> <span class="key">for</span> <span class="nam">h</span> <span class="key">in</span> <span class="nam">range</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">n_heads</span><span class="op">)</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t2381" href="#t2381">2381</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2382" href="#t2382">2382</a></span><span class="t"><span class="str">        Helper function to load in a 10K-20K dataset of elements from the model's training data</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2383" href="#t2383">2383</a></span><span class="t"><span class="str">        distribution.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2382" href="#t2382">2382</a></span><span class="t">    <span class="key">def</span> <span class="nam">load_sample_training_dataset</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2383" href="#t2383">2383</a></span><span class="t">        <span class="str">"""Load Sample Training Dataset.</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t2384" href="#t2384">2384</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2385" href="#t2385">2385</a></span><span class="t"><span class="str">        Wrapper around utils.get_dataset, which identifies the appropriate dataset the pretrained</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2386" href="#t2386">2386</a></span><span class="t"><span class="str">        models. Each dataset has a 'text' field, which contains the relevant info, some have several</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2387" href="#t2387">2387</a></span><span class="t"><span class="str">        meta data fields.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2388" href="#t2388">2388</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2389" href="#t2389">2389</a></span><span class="t"><span class="str">        Kwargs will be passed to utils.get_dataset (e.g. cache_dir to set download location)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2390" href="#t2390">2390</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2391" href="#t2391">2391</a></span><span class="t"><span class="str">        Notes:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2392" href="#t2392">2392</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2393" href="#t2393">2393</a></span><span class="t"><span class="str">        - PT-2's training data is not open source. OpenWebText is a replication (links with</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2394" href="#t2394">2394</a></span><span class="t"><span class="str">            >3 karma on Reddit)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2395" href="#t2395">2395</a></span><span class="t"><span class="str">        - OPT's training data is not open source, and is a mess of different things that is hard to</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2396" href="#t2396">2396</a></span><span class="t"><span class="str">          replicate. I default to the Pile, which covers some of it, but imperfectly.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2397" href="#t2397">2397</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2398" href="#t2398">2398</a></span><span class="t"><span class="str">        (Some models will have actually been trained on the data supplied here, for some it's from</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2399" href="#t2399">2399</a></span><span class="t"><span class="str">        the validation set).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2400" href="#t2400">2400</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2401" href="#t2401">2401</a></span><span class="t">        <span class="nam">model_dataset_map</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2402" href="#t2402">2402</a></span><span class="t">            <span class="str">"neel"</span><span class="op">:</span> <span class="str">"c4_code"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2403" href="#t2403">2403</a></span><span class="t">            <span class="str">"neel-solu-old"</span><span class="op">:</span> <span class="str">"pile"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2404" href="#t2404">2404</a></span><span class="t">            <span class="str">"GPT2LMHeadModel"</span><span class="op">:</span> <span class="str">"openwebtext"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2405" href="#t2405">2405</a></span><span class="t">            <span class="str">"GPTNeoForCausalLM"</span><span class="op">:</span> <span class="str">"pile"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2406" href="#t2406">2406</a></span><span class="t">            <span class="str">"GPTNeoXForCausalLM"</span><span class="op">:</span> <span class="str">"pile"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2407" href="#t2407">2407</a></span><span class="t">            <span class="str">"GPTJForCausalLM"</span><span class="op">:</span> <span class="str">"pile"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2408" href="#t2408">2408</a></span><span class="t">            <span class="str">"OPTForCausalLM"</span><span class="op">:</span> <span class="str">"pile"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2409" href="#t2409">2409</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2410" href="#t2410">2410</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="key">in</span> <span class="nam">model_dataset_map</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2411" href="#t2411">2411</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">dataset</span> <span class="op">=</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">get_dataset</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2412" href="#t2412">2412</a></span><span class="t">                <span class="nam">model_dataset_map</span><span class="op">[</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span><span class="op">]</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2413" href="#t2413">2413</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2414" href="#t2414">2414</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2415" href="#t2415">2415</a></span><span class="t">            <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2416" href="#t2416">2416</a></span><span class="t">                <span class="str">f"We do not have an available dataset for the relevant model: {self.cfg.original_architecture}"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2417" href="#t2417">2417</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2418" href="#t2418">2418</a></span><span class="t">        <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">dataset</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2419" href="#t2419">2419</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2420" href="#t2420">2420</a></span><span class="t">    <span class="key">def</span> <span class="nam">sample_datapoint</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2421" href="#t2421">2421</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2422" href="#t2422">2422</a></span><span class="t">        <span class="nam">tokenize</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2423" href="#t2423">2423</a></span><span class="t">        <span class="nam">prepend_bos</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Union</span><span class="op">[</span><span class="nam">bool</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2424" href="#t2424">2424</a></span><span class="t">        <span class="nam">padding_side</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Literal</span><span class="op">[</span><span class="str">"left"</span><span class="op">,</span> <span class="str">"right"</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2425" href="#t2425">2425</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Union</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"1 pos"</span><span class="op">]</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2426" href="#t2426">2426</a></span><span class="t">        <span class="str">"""Sample Data Point from Dataset.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2427" href="#t2427">2427</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2428" href="#t2428">2428</a></span><span class="t"><span class="str">        Helper function to randomly sample a data point from self.dataset, a small dataset from the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2429" href="#t2429">2429</a></span><span class="t"><span class="str">        data distribution the model was trained on.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2385" href="#t2385">2385</a></span><span class="t"><span class="str">        Helper function to load in a 10K-20K dataset of elements from the model's training data</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2386" href="#t2386">2386</a></span><span class="t"><span class="str">        distribution.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2387" href="#t2387">2387</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2388" href="#t2388">2388</a></span><span class="t"><span class="str">        Wrapper around utils.get_dataset, which identifies the appropriate dataset the pretrained</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2389" href="#t2389">2389</a></span><span class="t"><span class="str">        models. Each dataset has a 'text' field, which contains the relevant info, some have several</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2390" href="#t2390">2390</a></span><span class="t"><span class="str">        meta data fields.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2391" href="#t2391">2391</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2392" href="#t2392">2392</a></span><span class="t"><span class="str">        Kwargs will be passed to utils.get_dataset (e.g. cache_dir to set download location)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2393" href="#t2393">2393</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2394" href="#t2394">2394</a></span><span class="t"><span class="str">        Notes:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2395" href="#t2395">2395</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2396" href="#t2396">2396</a></span><span class="t"><span class="str">        - PT-2's training data is not open source. OpenWebText is a replication (links with</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2397" href="#t2397">2397</a></span><span class="t"><span class="str">            >3 karma on Reddit)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2398" href="#t2398">2398</a></span><span class="t"><span class="str">        - OPT's training data is not open source, and is a mess of different things that is hard to</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2399" href="#t2399">2399</a></span><span class="t"><span class="str">          replicate. I default to the Pile, which covers some of it, but imperfectly.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2400" href="#t2400">2400</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2401" href="#t2401">2401</a></span><span class="t"><span class="str">        (Some models will have actually been trained on the data supplied here, for some it's from</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2402" href="#t2402">2402</a></span><span class="t"><span class="str">        the validation set).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2403" href="#t2403">2403</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2404" href="#t2404">2404</a></span><span class="t">        <span class="nam">model_dataset_map</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2405" href="#t2405">2405</a></span><span class="t">            <span class="str">"neel"</span><span class="op">:</span> <span class="str">"c4_code"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2406" href="#t2406">2406</a></span><span class="t">            <span class="str">"neel-solu-old"</span><span class="op">:</span> <span class="str">"pile"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2407" href="#t2407">2407</a></span><span class="t">            <span class="str">"GPT2LMHeadModel"</span><span class="op">:</span> <span class="str">"openwebtext"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2408" href="#t2408">2408</a></span><span class="t">            <span class="str">"GPTNeoForCausalLM"</span><span class="op">:</span> <span class="str">"pile"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2409" href="#t2409">2409</a></span><span class="t">            <span class="str">"GPTNeoXForCausalLM"</span><span class="op">:</span> <span class="str">"pile"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2410" href="#t2410">2410</a></span><span class="t">            <span class="str">"GPTJForCausalLM"</span><span class="op">:</span> <span class="str">"pile"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2411" href="#t2411">2411</a></span><span class="t">            <span class="str">"OPTForCausalLM"</span><span class="op">:</span> <span class="str">"pile"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2412" href="#t2412">2412</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2413" href="#t2413">2413</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="key">in</span> <span class="nam">model_dataset_map</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2414" href="#t2414">2414</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">dataset</span> <span class="op">=</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">get_dataset</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2415" href="#t2415">2415</a></span><span class="t">                <span class="nam">model_dataset_map</span><span class="op">[</span><span class="nam">self</span><span class="op">.</span><span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span><span class="op">]</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2416" href="#t2416">2416</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2417" href="#t2417">2417</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2418" href="#t2418">2418</a></span><span class="t">            <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2419" href="#t2419">2419</a></span><span class="t">                <span class="str">f"We do not have an available dataset for the relevant model: {self.cfg.original_architecture}"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2420" href="#t2420">2420</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2421" href="#t2421">2421</a></span><span class="t">        <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">dataset</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2422" href="#t2422">2422</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2423" href="#t2423">2423</a></span><span class="t">    <span class="key">def</span> <span class="nam">sample_datapoint</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2424" href="#t2424">2424</a></span><span class="t">        <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2425" href="#t2425">2425</a></span><span class="t">        <span class="nam">tokenize</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2426" href="#t2426">2426</a></span><span class="t">        <span class="nam">prepend_bos</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Union</span><span class="op">[</span><span class="nam">bool</span><span class="op">,</span> <span class="key">None</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2427" href="#t2427">2427</a></span><span class="t">        <span class="nam">padding_side</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">Literal</span><span class="op">[</span><span class="str">"left"</span><span class="op">,</span> <span class="str">"right"</span><span class="op">]</span><span class="op">]</span> <span class="op">=</span> <span class="nam">USE_DEFAULT_VALUE</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2428" href="#t2428">2428</a></span><span class="t">    <span class="op">)</span> <span class="op">-></span> <span class="nam">Union</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">Float</span><span class="op">[</span><span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">,</span> <span class="str">"1 pos"</span><span class="op">]</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2429" href="#t2429">2429</a></span><span class="t">        <span class="str">"""Sample Data Point from Dataset.</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t2430" href="#t2430">2430</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2431" href="#t2431">2431</a></span><span class="t"><span class="str">        Implicitly calls self.load_sample_training_dataset if it hasn't already been called. Only</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2432" href="#t2432">2432</a></span><span class="t"><span class="str">        works for pretrained models with an associated dataset. But you can manually replace</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2433" href="#t2433">2433</a></span><span class="t"><span class="str">        self.dataset with a dataset of your choice if you want.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2434" href="#t2434">2434</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2435" href="#t2435">2435</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2436" href="#t2436">2436</a></span><span class="t"><span class="str">            tokenize (bool): Whether to return tokens (instead of text). Defaults to False. Note</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2437" href="#t2437">2437</a></span><span class="t"><span class="str">                that the returned tokens will be automatically truncated to the model's max context</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2438" href="#t2438">2438</a></span><span class="t"><span class="str">                size.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2439" href="#t2439">2439</a></span><span class="t"><span class="str">            prepend_bos (bool, optional): Overrides self.cfg.default_prepend_bos. Whether to prepend</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2440" href="#t2440">2440</a></span><span class="t"><span class="str">                the BOS token to the input (applicable when input is a string). Defaults to None,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2441" href="#t2441">2441</a></span><span class="t"><span class="str">                implying usage of self.cfg.default_prepend_bos (default is True unless specified</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2442" href="#t2442">2442</a></span><span class="t"><span class="str">                otherwise). Pass True or False to override the default.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2443" href="#t2443">2443</a></span><span class="t"><span class="str">            padding_side (Union[Literal["left", "right"], None], optional): Overrides</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2444" href="#t2444">2444</a></span><span class="t"><span class="str">                self.tokenizer.padding_side. Specifies which side to pad when tokenizing multiple</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2445" href="#t2445">2445</a></span><span class="t"><span class="str">                strings of different lengths.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2446" href="#t2446">2446</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2447" href="#t2447">2447</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">dataset</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2448" href="#t2448">2448</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">load_sample_training_dataset</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2449" href="#t2449">2449</a></span><span class="t">        <span class="key">assert</span> <span class="nam">self</span><span class="op">.</span><span class="nam">dataset</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span>  <span class="com"># keep mypy happy</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2450" href="#t2450">2450</a></span><span class="t">        <span class="nam">sample_dataset_size</span> <span class="op">=</span> <span class="nam">len</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">dataset</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2451" href="#t2451">2451</a></span><span class="t">        <span class="nam">index</span> <span class="op">=</span> <span class="nam">np</span><span class="op">.</span><span class="nam">random</span><span class="op">.</span><span class="nam">randint</span><span class="op">(</span><span class="num">0</span><span class="op">,</span> <span class="nam">sample_dataset_size</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2452" href="#t2452">2452</a></span><span class="t">        <span class="key">if</span> <span class="key">not</span> <span class="nam">tokenize</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t2453" href="#t2453">2453</a></span><span class="t">            <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">dataset</span><span class="op">[</span><span class="nam">index</span><span class="op">]</span><span class="op">[</span><span class="str">"text"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2454" href="#t2454">2454</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t2455" href="#t2455">2455</a></span><span class="t">            <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">to_tokens</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2456" href="#t2456">2456</a></span><span class="t">                <span class="nam">self</span><span class="op">.</span><span class="nam">dataset</span><span class="op">[</span><span class="nam">index</span><span class="op">]</span><span class="op">[</span><span class="str">"text"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2457" href="#t2457">2457</a></span><span class="t">                <span class="nam">prepend_bos</span><span class="op">=</span><span class="nam">prepend_bos</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2458" href="#t2458">2458</a></span><span class="t">                <span class="nam">padding_side</span><span class="op">=</span><span class="nam">padding_side</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2459" href="#t2459">2459</a></span><span class="t">                <span class="nam">truncate</span><span class="op">=</span><span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t2460" href="#t2460">2460</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2431" href="#t2431">2431</a></span><span class="t"><span class="str">        Helper function to randomly sample a data point from self.dataset, a small dataset from the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2432" href="#t2432">2432</a></span><span class="t"><span class="str">        data distribution the model was trained on.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2433" href="#t2433">2433</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2434" href="#t2434">2434</a></span><span class="t"><span class="str">        Implicitly calls self.load_sample_training_dataset if it hasn't already been called. Only</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2435" href="#t2435">2435</a></span><span class="t"><span class="str">        works for pretrained models with an associated dataset. But you can manually replace</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2436" href="#t2436">2436</a></span><span class="t"><span class="str">        self.dataset with a dataset of your choice if you want.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2437" href="#t2437">2437</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2438" href="#t2438">2438</a></span><span class="t"><span class="str">        Args:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2439" href="#t2439">2439</a></span><span class="t"><span class="str">            tokenize (bool): Whether to return tokens (instead of text). Defaults to False. Note</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2440" href="#t2440">2440</a></span><span class="t"><span class="str">                that the returned tokens will be automatically truncated to the model's max context</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2441" href="#t2441">2441</a></span><span class="t"><span class="str">                size.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2442" href="#t2442">2442</a></span><span class="t"><span class="str">            prepend_bos (bool, optional): Overrides self.cfg.default_prepend_bos. Whether to prepend</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2443" href="#t2443">2443</a></span><span class="t"><span class="str">                the BOS token to the input (applicable when input is a string). Defaults to None,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2444" href="#t2444">2444</a></span><span class="t"><span class="str">                implying usage of self.cfg.default_prepend_bos (default is True unless specified</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2445" href="#t2445">2445</a></span><span class="t"><span class="str">                otherwise). Pass True or False to override the default.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2446" href="#t2446">2446</a></span><span class="t"><span class="str">            padding_side (Union[Literal["left", "right"], None], optional): Overrides</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2447" href="#t2447">2447</a></span><span class="t"><span class="str">                self.tokenizer.padding_side. Specifies which side to pad when tokenizing multiple</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2448" href="#t2448">2448</a></span><span class="t"><span class="str">                strings of different lengths.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2449" href="#t2449">2449</a></span><span class="t"><span class="str">        """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2450" href="#t2450">2450</a></span><span class="t">        <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">dataset</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2451" href="#t2451">2451</a></span><span class="t">            <span class="nam">self</span><span class="op">.</span><span class="nam">load_sample_training_dataset</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2452" href="#t2452">2452</a></span><span class="t">        <span class="key">assert</span> <span class="nam">self</span><span class="op">.</span><span class="nam">dataset</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span>  <span class="com"># keep mypy happy</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2453" href="#t2453">2453</a></span><span class="t">        <span class="nam">sample_dataset_size</span> <span class="op">=</span> <span class="nam">len</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">dataset</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2454" href="#t2454">2454</a></span><span class="t">        <span class="nam">index</span> <span class="op">=</span> <span class="nam">np</span><span class="op">.</span><span class="nam">random</span><span class="op">.</span><span class="nam">randint</span><span class="op">(</span><span class="num">0</span><span class="op">,</span> <span class="nam">sample_dataset_size</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2455" href="#t2455">2455</a></span><span class="t">        <span class="key">if</span> <span class="key">not</span> <span class="nam">tokenize</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t2456" href="#t2456">2456</a></span><span class="t">            <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">dataset</span><span class="op">[</span><span class="nam">index</span><span class="op">]</span><span class="op">[</span><span class="str">"text"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2457" href="#t2457">2457</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t2458" href="#t2458">2458</a></span><span class="t">            <span class="key">return</span> <span class="nam">self</span><span class="op">.</span><span class="nam">to_tokens</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2459" href="#t2459">2459</a></span><span class="t">                <span class="nam">self</span><span class="op">.</span><span class="nam">dataset</span><span class="op">[</span><span class="nam">index</span><span class="op">]</span><span class="op">[</span><span class="str">"text"</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2460" href="#t2460">2460</a></span><span class="t">                <span class="nam">prepend_bos</span><span class="op">=</span><span class="nam">prepend_bos</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2461" href="#t2461">2461</a></span><span class="t">                <span class="nam">padding_side</span><span class="op">=</span><span class="nam">padding_side</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2462" href="#t2462">2462</a></span><span class="t">                <span class="nam">truncate</span><span class="op">=</span><span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t2463" href="#t2463">2463</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
 </main>
 <footer>
     <div class="content">
@@ -2551,7 +2554,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_HookedTransformerConfig_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_af97b5493da09a14_SVDInterpreter_py.html b/_static/coverage/d_af97b5493da09a14_SVDInterpreter_py.html
index 2b5ae611b..89a9ec507 100644
--- a/_static/coverage/d_af97b5493da09a14_SVDInterpreter_py.html
+++ b/_static/coverage/d_af97b5493da09a14_SVDInterpreter_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14___init___py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -253,7 +253,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14___init___py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_af97b5493da09a14___init___py.html b/_static/coverage/d_af97b5493da09a14___init___py.html
index cb30e1048..cfc365061 100644
--- a/_static/coverage/d_af97b5493da09a14___init___py.html
+++ b/_static/coverage/d_af97b5493da09a14___init___py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831___init___py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -118,7 +118,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831___init___py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_af97b5493da09a14_evals_py.html b/_static/coverage/d_af97b5493da09a14_evals_py.html
index cfffd2b4e..d137dc284 100644
--- a/_static/coverage/d_af97b5493da09a14_evals_py.html
+++ b/_static/coverage/d_af97b5493da09a14_evals_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_65d4430f90bfb219_activation_function_factory_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -427,7 +427,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_65d4430f90bfb219_activation_function_factory_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_af97b5493da09a14_head_detector_py.html b/_static/coverage/d_af97b5493da09a14_head_detector_py.html
index b5d9805a2..ef6f5ffbe 100644
--- a/_static/coverage/d_af97b5493da09a14_head_detector_py.html
+++ b/_static/coverage/d_af97b5493da09a14_head_detector_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_hook_points_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -361,7 +361,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_hook_points_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_af97b5493da09a14_hook_points_py.html b/_static/coverage/d_af97b5493da09a14_hook_points_py.html
index b973350ea..6d99a482b 100644
--- a/_static/coverage/d_af97b5493da09a14_hook_points_py.html
+++ b/_static/coverage/d_af97b5493da09a14_hook_points_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_loading_from_pretrained_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -787,7 +787,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_loading_from_pretrained_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_af97b5493da09a14_loading_from_pretrained_py.html b/_static/coverage/d_af97b5493da09a14_loading_from_pretrained_py.html
index 31f3e2fc3..a05c5c20e 100644
--- a/_static/coverage/d_af97b5493da09a14_loading_from_pretrained_py.html
+++ b/_static/coverage/d_af97b5493da09a14_loading_from_pretrained_py.html
@@ -55,11 +55,11 @@ <h1>
             </div>
         </aside>
         <h2>
-            <span class="text">306 statements &nbsp;</span>
-            <button type="button" class="run button_toggle_run" value="run" data-shortcut="r" title="Toggle lines run">200<span class="text"> run</span></button>
-            <button type="button" class="mis show_mis button_toggle_mis" value="mis" data-shortcut="m" title="Toggle lines missing">106<span class="text"> missing</span></button>
+            <span class="text">308 statements &nbsp;</span>
+            <button type="button" class="run button_toggle_run" value="run" data-shortcut="r" title="Toggle lines run">201<span class="text"> run</span></button>
+            <button type="button" class="mis show_mis button_toggle_mis" value="mis" data-shortcut="m" title="Toggle lines missing">107<span class="text"> missing</span></button>
             <button type="button" class="exc show_exc button_toggle_exc" value="exc" data-shortcut="x" title="Toggle lines excluded">0<span class="text"> excluded</span></button>
-            <button type="button" class="par run show_par button_toggle_par" value="par" data-shortcut="p" title="Toggle lines partially run">46<span class="text"> partial</span></button>
+            <button type="button" class="par run show_par button_toggle_par" value="par" data-shortcut="p" title="Toggle lines partially run">47<span class="text"> partial</span></button>
         </h2>
         <p class="text">
             <a id="prevFileLink" class="nav" href="d_af97b5493da09a14_hook_points_py.html">&#xab; prev</a> &nbsp; &nbsp;
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_past_key_value_caching_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -1473,404 +1473,407 @@ <h2>
     <p class="pln"><span class="n"><a id="t1389" href="#t1389">1389</a></span><span class="t">    <span class="nam">n_devices</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">1</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1390" href="#t1390">1390</a></span><span class="t">    <span class="nam">default_prepend_bos</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1391" href="#t1391">1391</a></span><span class="t">    <span class="nam">dtype</span><span class="op">:</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">dtype</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">float32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1392" href="#t1392">1392</a></span><span class="t">    <span class="op">**</span><span class="nam">kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1393" href="#t1393">1393</a></span><span class="t"><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1394" href="#t1394">1394</a></span><span class="t">    <span class="str">"""Returns the pretrained model config as an HookedTransformerConfig object.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1395" href="#t1395">1395</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1396" href="#t1396">1396</a></span><span class="t"><span class="str">    There are two types of pretrained models: HuggingFace models (where</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1397" href="#t1397">1397</a></span><span class="t"><span class="str">    AutoModel and AutoConfig work), and models trained by me (NeelNanda) which</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1398" href="#t1398">1398</a></span><span class="t"><span class="str">    aren't as integrated with HuggingFace infrastructure.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1399" href="#t1399">1399</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1400" href="#t1400">1400</a></span><span class="t"><span class="str">    Args:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1401" href="#t1401">1401</a></span><span class="t"><span class="str">        model_name: The name of the model. This can be either the official</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1402" href="#t1402">1402</a></span><span class="t"><span class="str">            HuggingFace model name, or the name of a model trained by me</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1403" href="#t1403">1403</a></span><span class="t"><span class="str">            (NeelNanda).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1404" href="#t1404">1404</a></span><span class="t"><span class="str">        hf_cfg (dict, optional): Config of a loaded pretrained HF model,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1405" href="#t1405">1405</a></span><span class="t"><span class="str">            converted to a dictionary.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1406" href="#t1406">1406</a></span><span class="t"><span class="str">        checkpoint_index (int, optional): If loading from a</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1407" href="#t1407">1407</a></span><span class="t"><span class="str">            checkpoint, the index of the checkpoint to load. Defaults to None.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1408" href="#t1408">1408</a></span><span class="t"><span class="str">        checkpoint_value (int, optional): If loading from a checkpoint, the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1409" href="#t1409">1409</a></span><span class="t"><span class="str">        value of</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1410" href="#t1410">1410</a></span><span class="t"><span class="str">            the checkpoint to load, ie the step or token number (each model has</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1411" href="#t1411">1411</a></span><span class="t"><span class="str">            checkpoints labelled with exactly one of these). Defaults to None.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1412" href="#t1412">1412</a></span><span class="t"><span class="str">        fold_ln (bool, optional): Whether to fold the layer norm into the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1413" href="#t1413">1413</a></span><span class="t"><span class="str">            subsequent linear layers (see HookedTransformer.fold_layer_norm for</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1414" href="#t1414">1414</a></span><span class="t"><span class="str">            details). Defaults to False.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1415" href="#t1415">1415</a></span><span class="t"><span class="str">        device (str, optional): The device to load the model onto. By</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1416" href="#t1416">1416</a></span><span class="t"><span class="str">            default will load to CUDA if available, else CPU.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1417" href="#t1417">1417</a></span><span class="t"><span class="str">        n_devices (int, optional): The number of devices to split the model across. Defaults to 1.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1418" href="#t1418">1418</a></span><span class="t"><span class="str">        default_prepend_bos (bool, optional): Default behavior of whether to prepend the BOS token when the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1419" href="#t1419">1419</a></span><span class="t"><span class="str">            methods of HookedTransformer process input text to tokenize (only when input is a string).</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1420" href="#t1420">1420</a></span><span class="t"><span class="str">            Defaults to True - even for models not explicitly trained with this, heads often use the</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1421" href="#t1421">1421</a></span><span class="t"><span class="str">            first position as a resting position and accordingly lose information from the first token,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1422" href="#t1422">1422</a></span><span class="t"><span class="str">            so this empirically seems to give better results. To change the default behavior to False, pass in</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1423" href="#t1423">1423</a></span><span class="t"><span class="str">            default_prepend_bos=False. Note that you can also locally override the default behavior by passing</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1424" href="#t1424">1424</a></span><span class="t"><span class="str">            in prepend_bos=True/False when you call a method that processes the input string.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1425" href="#t1425">1425</a></span><span class="t"><span class="str">        dtype (torch.dtype, optional): The dtype to load the TransformerLens model in.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1426" href="#t1426">1426</a></span><span class="t"><span class="str">        kwargs: Other optional arguments passed to HuggingFace's from_pretrained.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1427" href="#t1427">1427</a></span><span class="t"><span class="str">            Also given to other HuggingFace functions when compatible.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1428" href="#t1428">1428</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1429" href="#t1429">1429</a></span><span class="t"><span class="str">    """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1430" href="#t1430">1430</a></span><span class="t">    <span class="key">if</span> <span class="nam">Path</span><span class="op">(</span><span class="nam">model_name</span><span class="op">)</span><span class="op">.</span><span class="nam">exists</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1430&#x202F;&#x219B;&#x202F;1432</span><span class="annotate long">line 1430 didn't jump to line 1432, because the condition on line 1430 was never true</span></span></p>
-    <p class="pln"><span class="n"><a id="t1431" href="#t1431">1431</a></span><span class="t">        <span class="com"># If the model_name is a path, it's a local model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1432" href="#t1432">1432</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="nam">convert_hf_model_config</span><span class="op">(</span><span class="nam">model_name</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1433" href="#t1433">1433</a></span><span class="t">        <span class="nam">official_model_name</span> <span class="op">=</span> <span class="nam">model_name</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1434" href="#t1434">1434</a></span><span class="t">    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1435" href="#t1435">1435</a></span><span class="t">        <span class="nam">official_model_name</span> <span class="op">=</span> <span class="nam">get_official_model_name</span><span class="op">(</span><span class="nam">model_name</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1436" href="#t1436">1436</a></span><span class="t">    <span class="key">if</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1437" href="#t1437">1437</a></span><span class="t">        <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"NeelNanda"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1438" href="#t1438">1438</a></span><span class="t">        <span class="key">or</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"ArthurConmy"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1439" href="#t1439">1439</a></span><span class="t">        <span class="key">or</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"Baidicoot"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1440" href="#t1440">1440</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1441" href="#t1441">1441</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="nam">convert_neel_model_config</span><span class="op">(</span><span class="nam">official_model_name</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1442" href="#t1442">1442</a></span><span class="t">    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1443" href="#t1443">1443</a></span><span class="t">        <span class="key">if</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="nam">NEED_REMOTE_CODE_MODELS</span><span class="op">)</span> <span class="key">and</span> <span class="key">not</span> <span class="nam">kwargs</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span>&nbsp;</span><span class="r"><span class="annotate short">1443&#x202F;&#x219B;&#x202F;1446</span><span class="annotate long">line 1443 didn't jump to line 1446, because the condition on line 1443 was never true</span></span></p>
-    <p class="pln"><span class="n"><a id="t1444" href="#t1444">1444</a></span><span class="t">            <span class="str">"trust_remote_code"</span><span class="op">,</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1445" href="#t1445">1445</a></span><span class="t">        <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1446" href="#t1446">1446</a></span><span class="t">            <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1447" href="#t1447">1447</a></span><span class="t">                <span class="str">f"Loading model {official_model_name} requires setting trust_remote_code=True"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1448" href="#t1448">1448</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1449" href="#t1449">1449</a></span><span class="t">            <span class="nam">kwargs</span><span class="op">[</span><span class="str">"trust_remote_code"</span><span class="op">]</span> <span class="op">=</span> <span class="key">True</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1450" href="#t1450">1450</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="nam">convert_hf_model_config</span><span class="op">(</span><span class="nam">official_model_name</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1451" href="#t1451">1451</a></span><span class="t">    <span class="com"># Processing common to both model types</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1452" href="#t1452">1452</a></span><span class="t">    <span class="com"># Remove any prefix, saying the organization who made a model.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1453" href="#t1453">1453</a></span><span class="t">    <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"model_name"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">split</span><span class="op">(</span><span class="str">"/"</span><span class="op">)</span><span class="op">[</span><span class="op">-</span><span class="num">1</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1454" href="#t1454">1454</a></span><span class="t">    <span class="com"># Don't need to initialize weights, we're loading from pretrained</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1455" href="#t1455">1455</a></span><span class="t">    <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"init_weights"</span><span class="op">]</span> <span class="op">=</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1456" href="#t1456">1456</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1457" href="#t1457">1457</a></span><span class="t">    <span class="key">if</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1458" href="#t1458">1458</a></span><span class="t">        <span class="str">"positional_embedding_type"</span> <span class="key">in</span> <span class="nam">cfg_dict</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1459" href="#t1459">1459</a></span><span class="t">        <span class="key">and</span> <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"positional_embedding_type"</span><span class="op">]</span> <span class="op">==</span> <span class="str">"shortformer"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1460" href="#t1460">1460</a></span><span class="t">        <span class="key">and</span> <span class="nam">fold_ln</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1461" href="#t1461">1461</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1462" href="#t1462">1462</a></span><span class="t">        <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1463" href="#t1463">1463</a></span><span class="t">            <span class="str">"You tried to specify fold_ln=True for a shortformer model, but this can't be done! Setting fold_ln=False instead."</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1464" href="#t1464">1464</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1465" href="#t1465">1465</a></span><span class="t">        <span class="nam">fold_ln</span> <span class="op">=</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1466" href="#t1466">1466</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1467" href="#t1467">1467</a></span><span class="t">    <span class="key">if</span> <span class="nam">device</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1468" href="#t1468">1468</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"device"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">device</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1469" href="#t1469">1469</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1470" href="#t1470">1470</a></span><span class="t">    <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"dtype"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">dtype</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1471" href="#t1471">1471</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1472" href="#t1472">1472</a></span><span class="t">    <span class="key">if</span> <span class="nam">fold_ln</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1473" href="#t1473">1473</a></span><span class="t">        <span class="key">if</span> <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"normalization_type"</span><span class="op">]</span> <span class="key">in</span> <span class="op">[</span><span class="str">"LN"</span><span class="op">,</span> <span class="str">"LNPre"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1473&#x202F;&#x219B;&#x202F;1475</span><span class="annotate long">line 1473 didn't jump to line 1475, because the condition on line 1473 was never false</span></span></p>
-    <p class="run"><span class="n"><a id="t1474" href="#t1474">1474</a></span><span class="t">            <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"normalization_type"</span><span class="op">]</span> <span class="op">=</span> <span class="str">"LNPre"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1475" href="#t1475">1475</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"normalization_type"</span><span class="op">]</span> <span class="key">in</span> <span class="op">[</span><span class="str">"RMS"</span><span class="op">,</span> <span class="str">"RMSPre"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1476" href="#t1476">1476</a></span><span class="t">            <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"normalization_type"</span><span class="op">]</span> <span class="op">=</span> <span class="str">"RMSPre"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1477" href="#t1477">1477</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1478" href="#t1478">1478</a></span><span class="t">            <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span><span class="str">"Cannot fold in layer norm, normalization_type is not LN."</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1479" href="#t1479">1479</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1480" href="#t1480">1480</a></span><span class="t">    <span class="key">if</span> <span class="nam">checkpoint_index</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span> <span class="key">or</span> <span class="nam">checkpoint_value</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1480&#x202F;&#x219B;&#x202F;1481</span><span class="annotate long">line 1480 didn't jump to line 1481, because the condition on line 1480 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1481" href="#t1481">1481</a></span><span class="t">        <span class="nam">checkpoint_labels</span><span class="op">,</span> <span class="nam">checkpoint_label_type</span> <span class="op">=</span> <span class="nam">get_checkpoint_labels</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1482" href="#t1482">1482</a></span><span class="t">            <span class="nam">official_model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1483" href="#t1483">1483</a></span><span class="t">            <span class="op">**</span><span class="nam">kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1484" href="#t1484">1484</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1485" href="#t1485">1485</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"from_checkpoint"</span><span class="op">]</span> <span class="op">=</span> <span class="key">True</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1486" href="#t1486">1486</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"checkpoint_label_type"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">checkpoint_label_type</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1487" href="#t1487">1487</a></span><span class="t">        <span class="key">if</span> <span class="nam">checkpoint_index</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1488" href="#t1488">1488</a></span><span class="t">            <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"checkpoint_index"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">checkpoint_index</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1489" href="#t1489">1489</a></span><span class="t">            <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"checkpoint_value"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">checkpoint_labels</span><span class="op">[</span><span class="nam">checkpoint_index</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1490" href="#t1490">1490</a></span><span class="t">        <span class="key">elif</span> <span class="nam">checkpoint_value</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1491" href="#t1491">1491</a></span><span class="t">            <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1492" href="#t1492">1492</a></span><span class="t">                <span class="nam">checkpoint_value</span> <span class="key">in</span> <span class="nam">checkpoint_labels</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1493" href="#t1493">1493</a></span><span class="t">            <span class="op">)</span><span class="op">,</span> <span class="str">f"Checkpoint value {checkpoint_value} is not in list of available checkpoints"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1494" href="#t1494">1494</a></span><span class="t">            <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"checkpoint_value"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">checkpoint_value</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1495" href="#t1495">1495</a></span><span class="t">            <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"checkpoint_index"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">checkpoint_labels</span><span class="op">.</span><span class="nam">index</span><span class="op">(</span><span class="nam">checkpoint_value</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1496" href="#t1496">1496</a></span><span class="t">    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1497" href="#t1497">1497</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"from_checkpoint"</span><span class="op">]</span> <span class="op">=</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1498" href="#t1498">1498</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1499" href="#t1499">1499</a></span><span class="t">    <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"device"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">device</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1500" href="#t1500">1500</a></span><span class="t">    <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"n_devices"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">n_devices</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1501" href="#t1501">1501</a></span><span class="t">    <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"default_prepend_bos"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">default_prepend_bos</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1502" href="#t1502">1502</a></span><span class="t">    <span class="key">if</span> <span class="nam">hf_cfg</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1503" href="#t1503">1503</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"load_in_4bit"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">hf_cfg</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"quantization_config"</span><span class="op">,</span> <span class="op">{</span><span class="op">}</span><span class="op">)</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"load_in_4bit"</span><span class="op">,</span> <span class="key">False</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1504" href="#t1504">1504</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1505" href="#t1505">1505</a></span><span class="t">    <span class="nam">cfg</span> <span class="op">=</span> <span class="nam">HookedTransformerConfig</span><span class="op">.</span><span class="nam">from_dict</span><span class="op">(</span><span class="nam">cfg_dict</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1506" href="#t1506">1506</a></span><span class="t">    <span class="key">return</span> <span class="nam">cfg</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1392" href="#t1392">1392</a></span><span class="t">    <span class="nam">first_n_layers</span><span class="op">:</span> <span class="nam">Optional</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1393" href="#t1393">1393</a></span><span class="t">    <span class="op">**</span><span class="nam">kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1394" href="#t1394">1394</a></span><span class="t"><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1395" href="#t1395">1395</a></span><span class="t">    <span class="str">"""Returns the pretrained model config as an HookedTransformerConfig object.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1396" href="#t1396">1396</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1397" href="#t1397">1397</a></span><span class="t"><span class="str">    There are two types of pretrained models: HuggingFace models (where</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1398" href="#t1398">1398</a></span><span class="t"><span class="str">    AutoModel and AutoConfig work), and models trained by me (NeelNanda) which</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1399" href="#t1399">1399</a></span><span class="t"><span class="str">    aren't as integrated with HuggingFace infrastructure.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1400" href="#t1400">1400</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1401" href="#t1401">1401</a></span><span class="t"><span class="str">    Args:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1402" href="#t1402">1402</a></span><span class="t"><span class="str">        model_name: The name of the model. This can be either the official</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1403" href="#t1403">1403</a></span><span class="t"><span class="str">            HuggingFace model name, or the name of a model trained by me</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1404" href="#t1404">1404</a></span><span class="t"><span class="str">            (NeelNanda).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1405" href="#t1405">1405</a></span><span class="t"><span class="str">        hf_cfg (dict, optional): Config of a loaded pretrained HF model,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1406" href="#t1406">1406</a></span><span class="t"><span class="str">            converted to a dictionary.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1407" href="#t1407">1407</a></span><span class="t"><span class="str">        checkpoint_index (int, optional): If loading from a</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1408" href="#t1408">1408</a></span><span class="t"><span class="str">            checkpoint, the index of the checkpoint to load. Defaults to None.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1409" href="#t1409">1409</a></span><span class="t"><span class="str">        checkpoint_value (int, optional): If loading from a checkpoint, the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1410" href="#t1410">1410</a></span><span class="t"><span class="str">        value of</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1411" href="#t1411">1411</a></span><span class="t"><span class="str">            the checkpoint to load, ie the step or token number (each model has</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1412" href="#t1412">1412</a></span><span class="t"><span class="str">            checkpoints labelled with exactly one of these). Defaults to None.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1413" href="#t1413">1413</a></span><span class="t"><span class="str">        fold_ln (bool, optional): Whether to fold the layer norm into the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1414" href="#t1414">1414</a></span><span class="t"><span class="str">            subsequent linear layers (see HookedTransformer.fold_layer_norm for</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1415" href="#t1415">1415</a></span><span class="t"><span class="str">            details). Defaults to False.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1416" href="#t1416">1416</a></span><span class="t"><span class="str">        device (str, optional): The device to load the model onto. By</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1417" href="#t1417">1417</a></span><span class="t"><span class="str">            default will load to CUDA if available, else CPU.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1418" href="#t1418">1418</a></span><span class="t"><span class="str">        n_devices (int, optional): The number of devices to split the model across. Defaults to 1.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1419" href="#t1419">1419</a></span><span class="t"><span class="str">        default_prepend_bos (bool, optional): Default behavior of whether to prepend the BOS token when the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1420" href="#t1420">1420</a></span><span class="t"><span class="str">            methods of HookedTransformer process input text to tokenize (only when input is a string).</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1421" href="#t1421">1421</a></span><span class="t"><span class="str">            Defaults to True - even for models not explicitly trained with this, heads often use the</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1422" href="#t1422">1422</a></span><span class="t"><span class="str">            first position as a resting position and accordingly lose information from the first token,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1423" href="#t1423">1423</a></span><span class="t"><span class="str">            so this empirically seems to give better results. To change the default behavior to False, pass in</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1424" href="#t1424">1424</a></span><span class="t"><span class="str">            default_prepend_bos=False. Note that you can also locally override the default behavior by passing</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1425" href="#t1425">1425</a></span><span class="t"><span class="str">            in prepend_bos=True/False when you call a method that processes the input string.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1426" href="#t1426">1426</a></span><span class="t"><span class="str">        dtype (torch.dtype, optional): The dtype to load the TransformerLens model in.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1427" href="#t1427">1427</a></span><span class="t"><span class="str">        kwargs: Other optional arguments passed to HuggingFace's from_pretrained.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1428" href="#t1428">1428</a></span><span class="t"><span class="str">            Also given to other HuggingFace functions when compatible.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1429" href="#t1429">1429</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1430" href="#t1430">1430</a></span><span class="t"><span class="str">    """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1431" href="#t1431">1431</a></span><span class="t">    <span class="key">if</span> <span class="nam">Path</span><span class="op">(</span><span class="nam">model_name</span><span class="op">)</span><span class="op">.</span><span class="nam">exists</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1431&#x202F;&#x219B;&#x202F;1433</span><span class="annotate long">line 1431 didn't jump to line 1433, because the condition on line 1431 was never true</span></span></p>
+    <p class="pln"><span class="n"><a id="t1432" href="#t1432">1432</a></span><span class="t">        <span class="com"># If the model_name is a path, it's a local model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1433" href="#t1433">1433</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="nam">convert_hf_model_config</span><span class="op">(</span><span class="nam">model_name</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1434" href="#t1434">1434</a></span><span class="t">        <span class="nam">official_model_name</span> <span class="op">=</span> <span class="nam">model_name</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1435" href="#t1435">1435</a></span><span class="t">    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1436" href="#t1436">1436</a></span><span class="t">        <span class="nam">official_model_name</span> <span class="op">=</span> <span class="nam">get_official_model_name</span><span class="op">(</span><span class="nam">model_name</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1437" href="#t1437">1437</a></span><span class="t">    <span class="key">if</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1438" href="#t1438">1438</a></span><span class="t">        <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"NeelNanda"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1439" href="#t1439">1439</a></span><span class="t">        <span class="key">or</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"ArthurConmy"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1440" href="#t1440">1440</a></span><span class="t">        <span class="key">or</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"Baidicoot"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1441" href="#t1441">1441</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1442" href="#t1442">1442</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="nam">convert_neel_model_config</span><span class="op">(</span><span class="nam">official_model_name</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1443" href="#t1443">1443</a></span><span class="t">    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1444" href="#t1444">1444</a></span><span class="t">        <span class="key">if</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="nam">NEED_REMOTE_CODE_MODELS</span><span class="op">)</span> <span class="key">and</span> <span class="key">not</span> <span class="nam">kwargs</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span>&nbsp;</span><span class="r"><span class="annotate short">1444&#x202F;&#x219B;&#x202F;1447</span><span class="annotate long">line 1444 didn't jump to line 1447, because the condition on line 1444 was never true</span></span></p>
+    <p class="pln"><span class="n"><a id="t1445" href="#t1445">1445</a></span><span class="t">            <span class="str">"trust_remote_code"</span><span class="op">,</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1446" href="#t1446">1446</a></span><span class="t">        <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1447" href="#t1447">1447</a></span><span class="t">            <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1448" href="#t1448">1448</a></span><span class="t">                <span class="str">f"Loading model {official_model_name} requires setting trust_remote_code=True"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1449" href="#t1449">1449</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1450" href="#t1450">1450</a></span><span class="t">            <span class="nam">kwargs</span><span class="op">[</span><span class="str">"trust_remote_code"</span><span class="op">]</span> <span class="op">=</span> <span class="key">True</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1451" href="#t1451">1451</a></span><span class="t">        <span class="nam">cfg_dict</span> <span class="op">=</span> <span class="nam">convert_hf_model_config</span><span class="op">(</span><span class="nam">official_model_name</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1452" href="#t1452">1452</a></span><span class="t">    <span class="com"># Processing common to both model types</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1453" href="#t1453">1453</a></span><span class="t">    <span class="com"># Remove any prefix, saying the organization who made a model.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1454" href="#t1454">1454</a></span><span class="t">    <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"model_name"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">split</span><span class="op">(</span><span class="str">"/"</span><span class="op">)</span><span class="op">[</span><span class="op">-</span><span class="num">1</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1455" href="#t1455">1455</a></span><span class="t">    <span class="com"># Don't need to initialize weights, we're loading from pretrained</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1456" href="#t1456">1456</a></span><span class="t">    <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"init_weights"</span><span class="op">]</span> <span class="op">=</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1457" href="#t1457">1457</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1458" href="#t1458">1458</a></span><span class="t">    <span class="key">if</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1459" href="#t1459">1459</a></span><span class="t">        <span class="str">"positional_embedding_type"</span> <span class="key">in</span> <span class="nam">cfg_dict</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1460" href="#t1460">1460</a></span><span class="t">        <span class="key">and</span> <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"positional_embedding_type"</span><span class="op">]</span> <span class="op">==</span> <span class="str">"shortformer"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1461" href="#t1461">1461</a></span><span class="t">        <span class="key">and</span> <span class="nam">fold_ln</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1462" href="#t1462">1462</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1463" href="#t1463">1463</a></span><span class="t">        <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1464" href="#t1464">1464</a></span><span class="t">            <span class="str">"You tried to specify fold_ln=True for a shortformer model, but this can't be done! Setting fold_ln=False instead."</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1465" href="#t1465">1465</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1466" href="#t1466">1466</a></span><span class="t">        <span class="nam">fold_ln</span> <span class="op">=</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1467" href="#t1467">1467</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1468" href="#t1468">1468</a></span><span class="t">    <span class="key">if</span> <span class="nam">device</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1469" href="#t1469">1469</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"device"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">device</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1470" href="#t1470">1470</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1471" href="#t1471">1471</a></span><span class="t">    <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"dtype"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">dtype</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1472" href="#t1472">1472</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1473" href="#t1473">1473</a></span><span class="t">    <span class="key">if</span> <span class="nam">fold_ln</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1474" href="#t1474">1474</a></span><span class="t">        <span class="key">if</span> <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"normalization_type"</span><span class="op">]</span> <span class="key">in</span> <span class="op">[</span><span class="str">"LN"</span><span class="op">,</span> <span class="str">"LNPre"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1474&#x202F;&#x219B;&#x202F;1476</span><span class="annotate long">line 1474 didn't jump to line 1476, because the condition on line 1474 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t1475" href="#t1475">1475</a></span><span class="t">            <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"normalization_type"</span><span class="op">]</span> <span class="op">=</span> <span class="str">"LNPre"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1476" href="#t1476">1476</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"normalization_type"</span><span class="op">]</span> <span class="key">in</span> <span class="op">[</span><span class="str">"RMS"</span><span class="op">,</span> <span class="str">"RMSPre"</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1477" href="#t1477">1477</a></span><span class="t">            <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"normalization_type"</span><span class="op">]</span> <span class="op">=</span> <span class="str">"RMSPre"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1478" href="#t1478">1478</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1479" href="#t1479">1479</a></span><span class="t">            <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span><span class="str">"Cannot fold in layer norm, normalization_type is not LN."</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1480" href="#t1480">1480</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1481" href="#t1481">1481</a></span><span class="t">    <span class="key">if</span> <span class="nam">checkpoint_index</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span> <span class="key">or</span> <span class="nam">checkpoint_value</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1481&#x202F;&#x219B;&#x202F;1482</span><span class="annotate long">line 1481 didn't jump to line 1482, because the condition on line 1481 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1482" href="#t1482">1482</a></span><span class="t">        <span class="nam">checkpoint_labels</span><span class="op">,</span> <span class="nam">checkpoint_label_type</span> <span class="op">=</span> <span class="nam">get_checkpoint_labels</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1483" href="#t1483">1483</a></span><span class="t">            <span class="nam">official_model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1484" href="#t1484">1484</a></span><span class="t">            <span class="op">**</span><span class="nam">kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1485" href="#t1485">1485</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1486" href="#t1486">1486</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"from_checkpoint"</span><span class="op">]</span> <span class="op">=</span> <span class="key">True</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1487" href="#t1487">1487</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"checkpoint_label_type"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">checkpoint_label_type</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1488" href="#t1488">1488</a></span><span class="t">        <span class="key">if</span> <span class="nam">checkpoint_index</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1489" href="#t1489">1489</a></span><span class="t">            <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"checkpoint_index"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">checkpoint_index</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1490" href="#t1490">1490</a></span><span class="t">            <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"checkpoint_value"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">checkpoint_labels</span><span class="op">[</span><span class="nam">checkpoint_index</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1491" href="#t1491">1491</a></span><span class="t">        <span class="key">elif</span> <span class="nam">checkpoint_value</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1492" href="#t1492">1492</a></span><span class="t">            <span class="key">assert</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1493" href="#t1493">1493</a></span><span class="t">                <span class="nam">checkpoint_value</span> <span class="key">in</span> <span class="nam">checkpoint_labels</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1494" href="#t1494">1494</a></span><span class="t">            <span class="op">)</span><span class="op">,</span> <span class="str">f"Checkpoint value {checkpoint_value} is not in list of available checkpoints"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1495" href="#t1495">1495</a></span><span class="t">            <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"checkpoint_value"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">checkpoint_value</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1496" href="#t1496">1496</a></span><span class="t">            <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"checkpoint_index"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">checkpoint_labels</span><span class="op">.</span><span class="nam">index</span><span class="op">(</span><span class="nam">checkpoint_value</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1497" href="#t1497">1497</a></span><span class="t">    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1498" href="#t1498">1498</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"from_checkpoint"</span><span class="op">]</span> <span class="op">=</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1499" href="#t1499">1499</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1500" href="#t1500">1500</a></span><span class="t">    <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"device"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">device</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1501" href="#t1501">1501</a></span><span class="t">    <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"n_devices"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">n_devices</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1502" href="#t1502">1502</a></span><span class="t">    <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"default_prepend_bos"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">default_prepend_bos</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1503" href="#t1503">1503</a></span><span class="t">    <span class="key">if</span> <span class="nam">hf_cfg</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1504" href="#t1504">1504</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"load_in_4bit"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">hf_cfg</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"quantization_config"</span><span class="op">,</span> <span class="op">{</span><span class="op">}</span><span class="op">)</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"load_in_4bit"</span><span class="op">,</span> <span class="key">False</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1505" href="#t1505">1505</a></span><span class="t">    <span class="key">if</span> <span class="nam">first_n_layers</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1505&#x202F;&#x219B;&#x202F;1506</span><span class="annotate long">line 1505 didn't jump to line 1506, because the condition on line 1505 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1506" href="#t1506">1506</a></span><span class="t">        <span class="nam">cfg_dict</span><span class="op">[</span><span class="str">"n_layers"</span><span class="op">]</span> <span class="op">=</span> <span class="nam">first_n_layers</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1507" href="#t1507">1507</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1508" href="#t1508">1508</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1509" href="#t1509">1509</a></span><span class="t"><span class="key">def</span> <span class="nam">get_num_params_of_pretrained</span><span class="op">(</span><span class="nam">model_name</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1510" href="#t1510">1510</a></span><span class="t">    <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1511" href="#t1511">1511</a></span><span class="t"><span class="str">    Returns the number of parameters of a pretrained model, used to filter to only run code for sufficiently small models.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1512" href="#t1512">1512</a></span><span class="t"><span class="str">    """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1513" href="#t1513">1513</a></span><span class="t">    <span class="nam">cfg</span> <span class="op">=</span> <span class="nam">get_pretrained_model_config</span><span class="op">(</span><span class="nam">model_name</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1514" href="#t1514">1514</a></span><span class="t">    <span class="key">return</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">n_params</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1515" href="#t1515">1515</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1516" href="#t1516">1516</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1517" href="#t1517">1517</a></span><span class="t"><span class="com"># %% Load checkpointed model state dicts</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1518" href="#t1518">1518</a></span><span class="t"><span class="com"># The steps for which there are checkpoints in the stanford crfm models</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1519" href="#t1519">1519</a></span><span class="t"><span class="nam">STANFORD_CRFM_CHECKPOINTS</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1520" href="#t1520">1520</a></span><span class="t">    <span class="nam">list</span><span class="op">(</span><span class="nam">range</span><span class="op">(</span><span class="num">0</span><span class="op">,</span> <span class="num">100</span><span class="op">,</span> <span class="num">10</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1521" href="#t1521">1521</a></span><span class="t">    <span class="op">+</span> <span class="nam">list</span><span class="op">(</span><span class="nam">range</span><span class="op">(</span><span class="num">100</span><span class="op">,</span> <span class="num">2000</span><span class="op">,</span> <span class="num">50</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1522" href="#t1522">1522</a></span><span class="t">    <span class="op">+</span> <span class="nam">list</span><span class="op">(</span><span class="nam">range</span><span class="op">(</span><span class="num">2000</span><span class="op">,</span> <span class="num">20000</span><span class="op">,</span> <span class="num">100</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1523" href="#t1523">1523</a></span><span class="t">    <span class="op">+</span> <span class="nam">list</span><span class="op">(</span><span class="nam">range</span><span class="op">(</span><span class="num">20000</span><span class="op">,</span> <span class="num">400000</span> <span class="op">+</span> <span class="num">1</span><span class="op">,</span> <span class="num">1000</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1524" href="#t1524">1524</a></span><span class="t"><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1525" href="#t1525">1525</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1526" href="#t1526">1526</a></span><span class="t"><span class="com"># Linearly spaced checkpoints for Pythia models, taken every 1000 steps.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1527" href="#t1527">1527</a></span><span class="t"><span class="com"># Batch size 2,097,152 tokens, so checkpoints every 2.1B tokens</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1528" href="#t1528">1528</a></span><span class="t"><span class="nam">PYTHIA_CHECKPOINTS</span> <span class="op">=</span> <span class="op">[</span><span class="num">0</span><span class="op">,</span> <span class="num">1</span><span class="op">,</span> <span class="num">2</span><span class="op">,</span> <span class="num">4</span><span class="op">,</span> <span class="num">8</span><span class="op">,</span> <span class="num">16</span><span class="op">,</span> <span class="num">32</span><span class="op">,</span> <span class="num">64</span><span class="op">,</span> <span class="num">128</span><span class="op">,</span> <span class="num">256</span><span class="op">,</span> <span class="num">512</span><span class="op">]</span> <span class="op">+</span> <span class="nam">list</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1529" href="#t1529">1529</a></span><span class="t">    <span class="nam">range</span><span class="op">(</span><span class="num">1000</span><span class="op">,</span> <span class="num">143000</span> <span class="op">+</span> <span class="num">1</span><span class="op">,</span> <span class="num">1000</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1530" href="#t1530">1530</a></span><span class="t"><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1531" href="#t1531">1531</a></span><span class="t"><span class="com"># Pythia V1 has log-spaced early checkpoints (see line above), but V0 doesn't</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1532" href="#t1532">1532</a></span><span class="t"><span class="nam">PYTHIA_V0_CHECKPOINTS</span> <span class="op">=</span> <span class="nam">list</span><span class="op">(</span><span class="nam">range</span><span class="op">(</span><span class="num">1000</span><span class="op">,</span> <span class="num">143000</span> <span class="op">+</span> <span class="num">1</span><span class="op">,</span> <span class="num">1000</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1533" href="#t1533">1533</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1534" href="#t1534">1534</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1535" href="#t1535">1535</a></span><span class="t"><span class="key">def</span> <span class="nam">get_checkpoint_labels</span><span class="op">(</span><span class="nam">model_name</span><span class="op">:</span> <span class="nam">str</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1536" href="#t1536">1536</a></span><span class="t">    <span class="str">"""Returns the checkpoint labels for a given model, and the label_type</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1537" href="#t1537">1537</a></span><span class="t"><span class="str">    (step or token). Raises an error for models that are not checkpointed."""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1538" href="#t1538">1538</a></span><span class="t">    <span class="nam">official_model_name</span> <span class="op">=</span> <span class="nam">get_official_model_name</span><span class="op">(</span><span class="nam">model_name</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1539" href="#t1539">1539</a></span><span class="t">    <span class="key">if</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"stanford-crfm/"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1540" href="#t1540">1540</a></span><span class="t">        <span class="key">return</span> <span class="nam">STANFORD_CRFM_CHECKPOINTS</span><span class="op">,</span> <span class="str">"step"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1541" href="#t1541">1541</a></span><span class="t">    <span class="key">elif</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"EleutherAI/pythia"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1542" href="#t1542">1542</a></span><span class="t">        <span class="key">if</span> <span class="str">"v0"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1543" href="#t1543">1543</a></span><span class="t">            <span class="key">return</span> <span class="nam">PYTHIA_V0_CHECKPOINTS</span><span class="op">,</span> <span class="str">"step"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1544" href="#t1544">1544</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1545" href="#t1545">1545</a></span><span class="t">            <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1546" href="#t1546">1546</a></span><span class="t">                <span class="str">"Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models."</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1547" href="#t1547">1547</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1548" href="#t1548">1548</a></span><span class="t">            <span class="key">return</span> <span class="nam">PYTHIA_CHECKPOINTS</span><span class="op">,</span> <span class="str">"step"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1549" href="#t1549">1549</a></span><span class="t">    <span class="key">elif</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"NeelNanda/"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1550" href="#t1550">1550</a></span><span class="t">        <span class="nam">api</span> <span class="op">=</span> <span class="nam">HfApi</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1551" href="#t1551">1551</a></span><span class="t">        <span class="nam">files_list</span> <span class="op">=</span> <span class="nam">api</span><span class="op">.</span><span class="nam">list_repo_files</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1552" href="#t1552">1552</a></span><span class="t">            <span class="nam">official_model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1553" href="#t1553">1553</a></span><span class="t">            <span class="op">**</span><span class="nam">utils</span><span class="op">.</span><span class="nam">select_compatible_kwargs</span><span class="op">(</span><span class="nam">kwargs</span><span class="op">,</span> <span class="nam">api</span><span class="op">.</span><span class="nam">list_repo_files</span><span class="op">)</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1554" href="#t1554">1554</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1555" href="#t1555">1555</a></span><span class="t">        <span class="nam">labels</span> <span class="op">=</span> <span class="op">[</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1556" href="#t1556">1556</a></span><span class="t">        <span class="key">for</span> <span class="nam">file_name</span> <span class="key">in</span> <span class="nam">files_list</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1557" href="#t1557">1557</a></span><span class="t">            <span class="nam">match</span> <span class="op">=</span> <span class="nam">re</span><span class="op">.</span><span class="nam">match</span><span class="op">(</span><span class="str">r"checkpoints/.*_(\d*)\.pth"</span><span class="op">,</span> <span class="nam">file_name</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1558" href="#t1558">1558</a></span><span class="t">            <span class="key">if</span> <span class="nam">match</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1559" href="#t1559">1559</a></span><span class="t">                <span class="nam">labels</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="nam">int</span><span class="op">(</span><span class="nam">match</span><span class="op">.</span><span class="nam">group</span><span class="op">(</span><span class="num">1</span><span class="op">)</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1560" href="#t1560">1560</a></span><span class="t">        <span class="key">if</span> <span class="nam">labels</span><span class="op">[</span><span class="op">-</span><span class="num">1</span><span class="op">]</span> <span class="op">></span> <span class="num">1e9</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1561" href="#t1561">1561</a></span><span class="t">            <span class="nam">label_type</span> <span class="op">=</span> <span class="str">"token"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1562" href="#t1562">1562</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1563" href="#t1563">1563</a></span><span class="t">            <span class="nam">label_type</span> <span class="op">=</span> <span class="str">"step"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1564" href="#t1564">1564</a></span><span class="t">        <span class="key">return</span> <span class="nam">labels</span><span class="op">,</span> <span class="nam">label_type</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1565" href="#t1565">1565</a></span><span class="t">    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1566" href="#t1566">1566</a></span><span class="t">        <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span><span class="str">f"Model {official_model_name} is not checkpointed."</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1567" href="#t1567">1567</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1568" href="#t1568">1568</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1569" href="#t1569">1569</a></span><span class="t"><span class="com"># %% Loading state dicts</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1570" href="#t1570">1570</a></span><span class="t"><span class="key">def</span> <span class="nam">get_pretrained_state_dict</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1571" href="#t1571">1571</a></span><span class="t">    <span class="nam">official_model_name</span><span class="op">:</span> <span class="nam">str</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1572" href="#t1572">1572</a></span><span class="t">    <span class="nam">cfg</span><span class="op">:</span> <span class="nam">HookedTransformerConfig</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1573" href="#t1573">1573</a></span><span class="t">    <span class="nam">hf_model</span><span class="op">=</span><span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1574" href="#t1574">1574</a></span><span class="t">    <span class="nam">dtype</span><span class="op">:</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">dtype</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">float32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1575" href="#t1575">1575</a></span><span class="t">    <span class="op">**</span><span class="nam">kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1576" href="#t1576">1576</a></span><span class="t"><span class="op">)</span> <span class="op">-></span> <span class="nam">Dict</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1577" href="#t1577">1577</a></span><span class="t">    <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1578" href="#t1578">1578</a></span><span class="t"><span class="str">    Loads in the model weights for a pretrained model, and processes them to</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1579" href="#t1579">1579</a></span><span class="t"><span class="str">    have the HookedTransformer parameter names and shapes. Supports checkpointed</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1580" href="#t1580">1580</a></span><span class="t"><span class="str">    models (and expects the checkpoint info to be stored in the config object)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1581" href="#t1581">1581</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1582" href="#t1582">1582</a></span><span class="t"><span class="str">    hf_model: Optionally, a HuggingFace model object. If provided, we will use</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1583" href="#t1583">1583</a></span><span class="t"><span class="str">        these weights rather than reloading the model.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1584" href="#t1584">1584</a></span><span class="t"><span class="str">    dtype: The dtype to load the HuggingFace model in.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1585" href="#t1585">1585</a></span><span class="t"><span class="str">    kwargs: Other optional arguments passed to HuggingFace's from_pretrained.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1586" href="#t1586">1586</a></span><span class="t"><span class="str">        Also given to other HuggingFace functions when compatible.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1587" href="#t1587">1587</a></span><span class="t"><span class="str">    """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1588" href="#t1588">1588</a></span><span class="t">    <span class="key">if</span> <span class="str">"torch_dtype"</span> <span class="key">in</span> <span class="nam">kwargs</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1588&#x202F;&#x219B;&#x202F;1589</span><span class="annotate long">line 1588 didn't jump to line 1589, because the condition on line 1588 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1589" href="#t1589">1589</a></span><span class="t">        <span class="nam">dtype</span> <span class="op">=</span> <span class="nam">kwargs</span><span class="op">[</span><span class="str">"torch_dtype"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1590" href="#t1590">1590</a></span><span class="t">        <span class="key">del</span> <span class="nam">kwargs</span><span class="op">[</span><span class="str">"torch_dtype"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1591" href="#t1591">1591</a></span><span class="t">    <span class="key">if</span> <span class="nam">Path</span><span class="op">(</span><span class="nam">official_model_name</span><span class="op">)</span><span class="op">.</span><span class="nam">exists</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1591&#x202F;&#x219B;&#x202F;1592</span><span class="annotate long">line 1591 didn't jump to line 1592, because the condition on line 1591 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1592" href="#t1592">1592</a></span><span class="t">        <span class="nam">official_model_name</span> <span class="op">=</span> <span class="nam">str</span><span class="op">(</span><span class="nam">Path</span><span class="op">(</span><span class="nam">official_model_name</span><span class="op">)</span><span class="op">.</span><span class="nam">resolve</span><span class="op">(</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1593" href="#t1593">1593</a></span><span class="t">        <span class="nam">logging</span><span class="op">.</span><span class="nam">info</span><span class="op">(</span><span class="str">f"Loading model from local path {official_model_name}"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1594" href="#t1594">1594</a></span><span class="t">    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1595" href="#t1595">1595</a></span><span class="t">        <span class="nam">official_model_name</span> <span class="op">=</span> <span class="nam">get_official_model_name</span><span class="op">(</span><span class="nam">official_model_name</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1596" href="#t1596">1596</a></span><span class="t">    <span class="key">if</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="nam">NEED_REMOTE_CODE_MODELS</span><span class="op">)</span> <span class="key">and</span> <span class="key">not</span> <span class="nam">kwargs</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span>&nbsp;</span><span class="r"><span class="annotate short">1596&#x202F;&#x219B;&#x202F;1599</span><span class="annotate long">line 1596 didn't jump to line 1599, because the condition on line 1596 was never true</span></span></p>
-    <p class="pln"><span class="n"><a id="t1597" href="#t1597">1597</a></span><span class="t">        <span class="str">"trust_remote_code"</span><span class="op">,</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1598" href="#t1598">1598</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1599" href="#t1599">1599</a></span><span class="t">        <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1600" href="#t1600">1600</a></span><span class="t">            <span class="str">f"Loading model {official_model_name} state dict requires setting trust_remote_code=True"</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1601" href="#t1601">1601</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1602" href="#t1602">1602</a></span><span class="t">        <span class="nam">kwargs</span><span class="op">[</span><span class="str">"trust_remote_code"</span><span class="op">]</span> <span class="op">=</span> <span class="key">True</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1603" href="#t1603">1603</a></span><span class="t">    <span class="key">if</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1604" href="#t1604">1604</a></span><span class="t">        <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"NeelNanda"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1605" href="#t1605">1605</a></span><span class="t">        <span class="key">or</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"ArthurConmy"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1606" href="#t1606">1606</a></span><span class="t">        <span class="key">or</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"Baidicoot"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1607" href="#t1607">1607</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1608" href="#t1608">1608</a></span><span class="t">        <span class="nam">api</span> <span class="op">=</span> <span class="nam">HfApi</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1609" href="#t1609">1609</a></span><span class="t">        <span class="nam">repo_files</span> <span class="op">=</span> <span class="nam">api</span><span class="op">.</span><span class="nam">list_repo_files</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1610" href="#t1610">1610</a></span><span class="t">            <span class="nam">official_model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1611" href="#t1611">1611</a></span><span class="t">            <span class="op">**</span><span class="nam">utils</span><span class="op">.</span><span class="nam">select_compatible_kwargs</span><span class="op">(</span><span class="nam">kwargs</span><span class="op">,</span> <span class="nam">api</span><span class="op">.</span><span class="nam">list_repo_files</span><span class="op">)</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1612" href="#t1612">1612</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1613" href="#t1613">1613</a></span><span class="t">        <span class="key">if</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">from_checkpoint</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1613&#x202F;&#x219B;&#x202F;1614</span><span class="annotate long">line 1613 didn't jump to line 1614, because the condition on line 1613 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1614" href="#t1614">1614</a></span><span class="t">            <span class="nam">file_name</span> <span class="op">=</span> <span class="nam">list</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1615" href="#t1615">1615</a></span><span class="t">                <span class="nam">filter</span><span class="op">(</span><span class="key">lambda</span> <span class="nam">x</span><span class="op">:</span> <span class="nam">x</span><span class="op">.</span><span class="nam">endswith</span><span class="op">(</span><span class="str">f"{cfg.checkpoint_value}.pth"</span><span class="op">)</span><span class="op">,</span> <span class="nam">repo_files</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1616" href="#t1616">1616</a></span><span class="t">            <span class="op">)</span><span class="op">[</span><span class="num">0</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1617" href="#t1617">1617</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1618" href="#t1618">1618</a></span><span class="t">            <span class="nam">file_name</span> <span class="op">=</span> <span class="nam">list</span><span class="op">(</span><span class="nam">filter</span><span class="op">(</span><span class="key">lambda</span> <span class="nam">x</span><span class="op">:</span> <span class="nam">x</span><span class="op">.</span><span class="nam">endswith</span><span class="op">(</span><span class="str">"final.pth"</span><span class="op">)</span><span class="op">,</span> <span class="nam">repo_files</span><span class="op">)</span><span class="op">)</span><span class="op">[</span><span class="num">0</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1619" href="#t1619">1619</a></span><span class="t">        <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">download_file_from_hf</span><span class="op">(</span><span class="nam">official_model_name</span><span class="op">,</span> <span class="nam">file_name</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1620" href="#t1620">1620</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1621" href="#t1621">1621</a></span><span class="t">        <span class="com"># Convert to dtype</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1622" href="#t1622">1622</a></span><span class="t">        <span class="nam">state_dict</span> <span class="op">=</span> <span class="op">{</span><span class="nam">k</span><span class="op">:</span> <span class="nam">v</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">dtype</span><span class="op">)</span> <span class="key">for</span> <span class="nam">k</span><span class="op">,</span> <span class="nam">v</span> <span class="key">in</span> <span class="nam">state_dict</span><span class="op">.</span><span class="nam">items</span><span class="op">(</span><span class="op">)</span><span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1508" href="#t1508">1508</a></span><span class="t">    <span class="nam">cfg</span> <span class="op">=</span> <span class="nam">HookedTransformerConfig</span><span class="op">.</span><span class="nam">from_dict</span><span class="op">(</span><span class="nam">cfg_dict</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1509" href="#t1509">1509</a></span><span class="t">    <span class="key">return</span> <span class="nam">cfg</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1510" href="#t1510">1510</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1511" href="#t1511">1511</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1512" href="#t1512">1512</a></span><span class="t"><span class="key">def</span> <span class="nam">get_num_params_of_pretrained</span><span class="op">(</span><span class="nam">model_name</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1513" href="#t1513">1513</a></span><span class="t">    <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1514" href="#t1514">1514</a></span><span class="t"><span class="str">    Returns the number of parameters of a pretrained model, used to filter to only run code for sufficiently small models.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1515" href="#t1515">1515</a></span><span class="t"><span class="str">    """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1516" href="#t1516">1516</a></span><span class="t">    <span class="nam">cfg</span> <span class="op">=</span> <span class="nam">get_pretrained_model_config</span><span class="op">(</span><span class="nam">model_name</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1517" href="#t1517">1517</a></span><span class="t">    <span class="key">return</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">n_params</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1518" href="#t1518">1518</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1519" href="#t1519">1519</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1520" href="#t1520">1520</a></span><span class="t"><span class="com"># %% Load checkpointed model state dicts</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1521" href="#t1521">1521</a></span><span class="t"><span class="com"># The steps for which there are checkpoints in the stanford crfm models</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1522" href="#t1522">1522</a></span><span class="t"><span class="nam">STANFORD_CRFM_CHECKPOINTS</span> <span class="op">=</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1523" href="#t1523">1523</a></span><span class="t">    <span class="nam">list</span><span class="op">(</span><span class="nam">range</span><span class="op">(</span><span class="num">0</span><span class="op">,</span> <span class="num">100</span><span class="op">,</span> <span class="num">10</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1524" href="#t1524">1524</a></span><span class="t">    <span class="op">+</span> <span class="nam">list</span><span class="op">(</span><span class="nam">range</span><span class="op">(</span><span class="num">100</span><span class="op">,</span> <span class="num">2000</span><span class="op">,</span> <span class="num">50</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1525" href="#t1525">1525</a></span><span class="t">    <span class="op">+</span> <span class="nam">list</span><span class="op">(</span><span class="nam">range</span><span class="op">(</span><span class="num">2000</span><span class="op">,</span> <span class="num">20000</span><span class="op">,</span> <span class="num">100</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1526" href="#t1526">1526</a></span><span class="t">    <span class="op">+</span> <span class="nam">list</span><span class="op">(</span><span class="nam">range</span><span class="op">(</span><span class="num">20000</span><span class="op">,</span> <span class="num">400000</span> <span class="op">+</span> <span class="num">1</span><span class="op">,</span> <span class="num">1000</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1527" href="#t1527">1527</a></span><span class="t"><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1528" href="#t1528">1528</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1529" href="#t1529">1529</a></span><span class="t"><span class="com"># Linearly spaced checkpoints for Pythia models, taken every 1000 steps.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1530" href="#t1530">1530</a></span><span class="t"><span class="com"># Batch size 2,097,152 tokens, so checkpoints every 2.1B tokens</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1531" href="#t1531">1531</a></span><span class="t"><span class="nam">PYTHIA_CHECKPOINTS</span> <span class="op">=</span> <span class="op">[</span><span class="num">0</span><span class="op">,</span> <span class="num">1</span><span class="op">,</span> <span class="num">2</span><span class="op">,</span> <span class="num">4</span><span class="op">,</span> <span class="num">8</span><span class="op">,</span> <span class="num">16</span><span class="op">,</span> <span class="num">32</span><span class="op">,</span> <span class="num">64</span><span class="op">,</span> <span class="num">128</span><span class="op">,</span> <span class="num">256</span><span class="op">,</span> <span class="num">512</span><span class="op">]</span> <span class="op">+</span> <span class="nam">list</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1532" href="#t1532">1532</a></span><span class="t">    <span class="nam">range</span><span class="op">(</span><span class="num">1000</span><span class="op">,</span> <span class="num">143000</span> <span class="op">+</span> <span class="num">1</span><span class="op">,</span> <span class="num">1000</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1533" href="#t1533">1533</a></span><span class="t"><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1534" href="#t1534">1534</a></span><span class="t"><span class="com"># Pythia V1 has log-spaced early checkpoints (see line above), but V0 doesn't</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1535" href="#t1535">1535</a></span><span class="t"><span class="nam">PYTHIA_V0_CHECKPOINTS</span> <span class="op">=</span> <span class="nam">list</span><span class="op">(</span><span class="nam">range</span><span class="op">(</span><span class="num">1000</span><span class="op">,</span> <span class="num">143000</span> <span class="op">+</span> <span class="num">1</span><span class="op">,</span> <span class="num">1000</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1536" href="#t1536">1536</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1537" href="#t1537">1537</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1538" href="#t1538">1538</a></span><span class="t"><span class="key">def</span> <span class="nam">get_checkpoint_labels</span><span class="op">(</span><span class="nam">model_name</span><span class="op">:</span> <span class="nam">str</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1539" href="#t1539">1539</a></span><span class="t">    <span class="str">"""Returns the checkpoint labels for a given model, and the label_type</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1540" href="#t1540">1540</a></span><span class="t"><span class="str">    (step or token). Raises an error for models that are not checkpointed."""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1541" href="#t1541">1541</a></span><span class="t">    <span class="nam">official_model_name</span> <span class="op">=</span> <span class="nam">get_official_model_name</span><span class="op">(</span><span class="nam">model_name</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1542" href="#t1542">1542</a></span><span class="t">    <span class="key">if</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"stanford-crfm/"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1543" href="#t1543">1543</a></span><span class="t">        <span class="key">return</span> <span class="nam">STANFORD_CRFM_CHECKPOINTS</span><span class="op">,</span> <span class="str">"step"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1544" href="#t1544">1544</a></span><span class="t">    <span class="key">elif</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"EleutherAI/pythia"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1545" href="#t1545">1545</a></span><span class="t">        <span class="key">if</span> <span class="str">"v0"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1546" href="#t1546">1546</a></span><span class="t">            <span class="key">return</span> <span class="nam">PYTHIA_V0_CHECKPOINTS</span><span class="op">,</span> <span class="str">"step"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1547" href="#t1547">1547</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1548" href="#t1548">1548</a></span><span class="t">            <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1549" href="#t1549">1549</a></span><span class="t">                <span class="str">"Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models."</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1550" href="#t1550">1550</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1551" href="#t1551">1551</a></span><span class="t">            <span class="key">return</span> <span class="nam">PYTHIA_CHECKPOINTS</span><span class="op">,</span> <span class="str">"step"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1552" href="#t1552">1552</a></span><span class="t">    <span class="key">elif</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"NeelNanda/"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1553" href="#t1553">1553</a></span><span class="t">        <span class="nam">api</span> <span class="op">=</span> <span class="nam">HfApi</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1554" href="#t1554">1554</a></span><span class="t">        <span class="nam">files_list</span> <span class="op">=</span> <span class="nam">api</span><span class="op">.</span><span class="nam">list_repo_files</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1555" href="#t1555">1555</a></span><span class="t">            <span class="nam">official_model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1556" href="#t1556">1556</a></span><span class="t">            <span class="op">**</span><span class="nam">utils</span><span class="op">.</span><span class="nam">select_compatible_kwargs</span><span class="op">(</span><span class="nam">kwargs</span><span class="op">,</span> <span class="nam">api</span><span class="op">.</span><span class="nam">list_repo_files</span><span class="op">)</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1557" href="#t1557">1557</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1558" href="#t1558">1558</a></span><span class="t">        <span class="nam">labels</span> <span class="op">=</span> <span class="op">[</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1559" href="#t1559">1559</a></span><span class="t">        <span class="key">for</span> <span class="nam">file_name</span> <span class="key">in</span> <span class="nam">files_list</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1560" href="#t1560">1560</a></span><span class="t">            <span class="nam">match</span> <span class="op">=</span> <span class="nam">re</span><span class="op">.</span><span class="nam">match</span><span class="op">(</span><span class="str">r"checkpoints/.*_(\d*)\.pth"</span><span class="op">,</span> <span class="nam">file_name</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1561" href="#t1561">1561</a></span><span class="t">            <span class="key">if</span> <span class="nam">match</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1562" href="#t1562">1562</a></span><span class="t">                <span class="nam">labels</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="nam">int</span><span class="op">(</span><span class="nam">match</span><span class="op">.</span><span class="nam">group</span><span class="op">(</span><span class="num">1</span><span class="op">)</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1563" href="#t1563">1563</a></span><span class="t">        <span class="key">if</span> <span class="nam">labels</span><span class="op">[</span><span class="op">-</span><span class="num">1</span><span class="op">]</span> <span class="op">></span> <span class="num">1e9</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1564" href="#t1564">1564</a></span><span class="t">            <span class="nam">label_type</span> <span class="op">=</span> <span class="str">"token"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1565" href="#t1565">1565</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1566" href="#t1566">1566</a></span><span class="t">            <span class="nam">label_type</span> <span class="op">=</span> <span class="str">"step"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1567" href="#t1567">1567</a></span><span class="t">        <span class="key">return</span> <span class="nam">labels</span><span class="op">,</span> <span class="nam">label_type</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1568" href="#t1568">1568</a></span><span class="t">    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1569" href="#t1569">1569</a></span><span class="t">        <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span><span class="str">f"Model {official_model_name} is not checkpointed."</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1570" href="#t1570">1570</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1571" href="#t1571">1571</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1572" href="#t1572">1572</a></span><span class="t"><span class="com"># %% Loading state dicts</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1573" href="#t1573">1573</a></span><span class="t"><span class="key">def</span> <span class="nam">get_pretrained_state_dict</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1574" href="#t1574">1574</a></span><span class="t">    <span class="nam">official_model_name</span><span class="op">:</span> <span class="nam">str</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1575" href="#t1575">1575</a></span><span class="t">    <span class="nam">cfg</span><span class="op">:</span> <span class="nam">HookedTransformerConfig</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1576" href="#t1576">1576</a></span><span class="t">    <span class="nam">hf_model</span><span class="op">=</span><span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1577" href="#t1577">1577</a></span><span class="t">    <span class="nam">dtype</span><span class="op">:</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">dtype</span> <span class="op">=</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">float32</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1578" href="#t1578">1578</a></span><span class="t">    <span class="op">**</span><span class="nam">kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1579" href="#t1579">1579</a></span><span class="t"><span class="op">)</span> <span class="op">-></span> <span class="nam">Dict</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">torch</span><span class="op">.</span><span class="nam">Tensor</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1580" href="#t1580">1580</a></span><span class="t">    <span class="str">"""</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1581" href="#t1581">1581</a></span><span class="t"><span class="str">    Loads in the model weights for a pretrained model, and processes them to</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1582" href="#t1582">1582</a></span><span class="t"><span class="str">    have the HookedTransformer parameter names and shapes. Supports checkpointed</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1583" href="#t1583">1583</a></span><span class="t"><span class="str">    models (and expects the checkpoint info to be stored in the config object)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1584" href="#t1584">1584</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1585" href="#t1585">1585</a></span><span class="t"><span class="str">    hf_model: Optionally, a HuggingFace model object. If provided, we will use</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1586" href="#t1586">1586</a></span><span class="t"><span class="str">        these weights rather than reloading the model.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1587" href="#t1587">1587</a></span><span class="t"><span class="str">    dtype: The dtype to load the HuggingFace model in.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1588" href="#t1588">1588</a></span><span class="t"><span class="str">    kwargs: Other optional arguments passed to HuggingFace's from_pretrained.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1589" href="#t1589">1589</a></span><span class="t"><span class="str">        Also given to other HuggingFace functions when compatible.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1590" href="#t1590">1590</a></span><span class="t"><span class="str">    """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1591" href="#t1591">1591</a></span><span class="t">    <span class="key">if</span> <span class="str">"torch_dtype"</span> <span class="key">in</span> <span class="nam">kwargs</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1591&#x202F;&#x219B;&#x202F;1592</span><span class="annotate long">line 1591 didn't jump to line 1592, because the condition on line 1591 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1592" href="#t1592">1592</a></span><span class="t">        <span class="nam">dtype</span> <span class="op">=</span> <span class="nam">kwargs</span><span class="op">[</span><span class="str">"torch_dtype"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1593" href="#t1593">1593</a></span><span class="t">        <span class="key">del</span> <span class="nam">kwargs</span><span class="op">[</span><span class="str">"torch_dtype"</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1594" href="#t1594">1594</a></span><span class="t">    <span class="key">if</span> <span class="nam">Path</span><span class="op">(</span><span class="nam">official_model_name</span><span class="op">)</span><span class="op">.</span><span class="nam">exists</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1594&#x202F;&#x219B;&#x202F;1595</span><span class="annotate long">line 1594 didn't jump to line 1595, because the condition on line 1594 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1595" href="#t1595">1595</a></span><span class="t">        <span class="nam">official_model_name</span> <span class="op">=</span> <span class="nam">str</span><span class="op">(</span><span class="nam">Path</span><span class="op">(</span><span class="nam">official_model_name</span><span class="op">)</span><span class="op">.</span><span class="nam">resolve</span><span class="op">(</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1596" href="#t1596">1596</a></span><span class="t">        <span class="nam">logging</span><span class="op">.</span><span class="nam">info</span><span class="op">(</span><span class="str">f"Loading model from local path {official_model_name}"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1597" href="#t1597">1597</a></span><span class="t">    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1598" href="#t1598">1598</a></span><span class="t">        <span class="nam">official_model_name</span> <span class="op">=</span> <span class="nam">get_official_model_name</span><span class="op">(</span><span class="nam">official_model_name</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1599" href="#t1599">1599</a></span><span class="t">    <span class="key">if</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="nam">NEED_REMOTE_CODE_MODELS</span><span class="op">)</span> <span class="key">and</span> <span class="key">not</span> <span class="nam">kwargs</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span>&nbsp;</span><span class="r"><span class="annotate short">1599&#x202F;&#x219B;&#x202F;1602</span><span class="annotate long">line 1599 didn't jump to line 1602, because the condition on line 1599 was never true</span></span></p>
+    <p class="pln"><span class="n"><a id="t1600" href="#t1600">1600</a></span><span class="t">        <span class="str">"trust_remote_code"</span><span class="op">,</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1601" href="#t1601">1601</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1602" href="#t1602">1602</a></span><span class="t">        <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1603" href="#t1603">1603</a></span><span class="t">            <span class="str">f"Loading model {official_model_name} state dict requires setting trust_remote_code=True"</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1604" href="#t1604">1604</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1605" href="#t1605">1605</a></span><span class="t">        <span class="nam">kwargs</span><span class="op">[</span><span class="str">"trust_remote_code"</span><span class="op">]</span> <span class="op">=</span> <span class="key">True</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1606" href="#t1606">1606</a></span><span class="t">    <span class="key">if</span> <span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1607" href="#t1607">1607</a></span><span class="t">        <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"NeelNanda"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1608" href="#t1608">1608</a></span><span class="t">        <span class="key">or</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"ArthurConmy"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1609" href="#t1609">1609</a></span><span class="t">        <span class="key">or</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"Baidicoot"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1610" href="#t1610">1610</a></span><span class="t">    <span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1611" href="#t1611">1611</a></span><span class="t">        <span class="nam">api</span> <span class="op">=</span> <span class="nam">HfApi</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1612" href="#t1612">1612</a></span><span class="t">        <span class="nam">repo_files</span> <span class="op">=</span> <span class="nam">api</span><span class="op">.</span><span class="nam">list_repo_files</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1613" href="#t1613">1613</a></span><span class="t">            <span class="nam">official_model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1614" href="#t1614">1614</a></span><span class="t">            <span class="op">**</span><span class="nam">utils</span><span class="op">.</span><span class="nam">select_compatible_kwargs</span><span class="op">(</span><span class="nam">kwargs</span><span class="op">,</span> <span class="nam">api</span><span class="op">.</span><span class="nam">list_repo_files</span><span class="op">)</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1615" href="#t1615">1615</a></span><span class="t">        <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1616" href="#t1616">1616</a></span><span class="t">        <span class="key">if</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">from_checkpoint</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1616&#x202F;&#x219B;&#x202F;1617</span><span class="annotate long">line 1616 didn't jump to line 1617, because the condition on line 1616 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1617" href="#t1617">1617</a></span><span class="t">            <span class="nam">file_name</span> <span class="op">=</span> <span class="nam">list</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1618" href="#t1618">1618</a></span><span class="t">                <span class="nam">filter</span><span class="op">(</span><span class="key">lambda</span> <span class="nam">x</span><span class="op">:</span> <span class="nam">x</span><span class="op">.</span><span class="nam">endswith</span><span class="op">(</span><span class="str">f"{cfg.checkpoint_value}.pth"</span><span class="op">)</span><span class="op">,</span> <span class="nam">repo_files</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1619" href="#t1619">1619</a></span><span class="t">            <span class="op">)</span><span class="op">[</span><span class="num">0</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1620" href="#t1620">1620</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1621" href="#t1621">1621</a></span><span class="t">            <span class="nam">file_name</span> <span class="op">=</span> <span class="nam">list</span><span class="op">(</span><span class="nam">filter</span><span class="op">(</span><span class="key">lambda</span> <span class="nam">x</span><span class="op">:</span> <span class="nam">x</span><span class="op">.</span><span class="nam">endswith</span><span class="op">(</span><span class="str">"final.pth"</span><span class="op">)</span><span class="op">,</span> <span class="nam">repo_files</span><span class="op">)</span><span class="op">)</span><span class="op">[</span><span class="num">0</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1622" href="#t1622">1622</a></span><span class="t">        <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">utils</span><span class="op">.</span><span class="nam">download_file_from_hf</span><span class="op">(</span><span class="nam">official_model_name</span><span class="op">,</span> <span class="nam">file_name</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1623" href="#t1623">1623</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1624" href="#t1624">1624</a></span><span class="t">        <span class="key">if</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"neel-solu-old"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1625" href="#t1625">1625</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_neel_solu_old_weights</span><span class="op">(</span><span class="nam">state_dict</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1626" href="#t1626">1626</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"mingpt"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1627" href="#t1627">1627</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_mingpt_weights</span><span class="op">(</span><span class="nam">state_dict</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1628" href="#t1628">1628</a></span><span class="t">        <span class="key">return</span> <span class="nam">state_dict</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1629" href="#t1629">1629</a></span><span class="t">    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1630" href="#t1630">1630</a></span><span class="t">        <span class="key">if</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">from_checkpoint</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1630&#x202F;&#x219B;&#x202F;1631</span><span class="annotate long">line 1630 didn't jump to line 1631, because the condition on line 1630 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1631" href="#t1631">1631</a></span><span class="t">            <span class="nam">huggingface_token</span> <span class="op">=</span> <span class="nam">os</span><span class="op">.</span><span class="nam">environ</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"HF_TOKEN"</span><span class="op">,</span> <span class="key">None</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1632" href="#t1632">1632</a></span><span class="t">            <span class="key">if</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"stanford-crfm"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1633" href="#t1633">1633</a></span><span class="t">                <span class="nam">hf_model</span> <span class="op">=</span> <span class="nam">AutoModelForCausalLM</span><span class="op">.</span><span class="nam">from_pretrained</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1634" href="#t1634">1634</a></span><span class="t">                    <span class="nam">official_model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1635" href="#t1635">1635</a></span><span class="t">                    <span class="nam">revision</span><span class="op">=</span><span class="str">f"checkpoint-{cfg.checkpoint_value}"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1636" href="#t1636">1636</a></span><span class="t">                    <span class="nam">torch_dtype</span><span class="op">=</span><span class="nam">dtype</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1637" href="#t1637">1637</a></span><span class="t">                    <span class="nam">token</span><span class="op">=</span><span class="nam">huggingface_token</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1638" href="#t1638">1638</a></span><span class="t">                    <span class="op">**</span><span class="nam">kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1639" href="#t1639">1639</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1640" href="#t1640">1640</a></span><span class="t">            <span class="key">elif</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"EleutherAI/pythia"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1641" href="#t1641">1641</a></span><span class="t">                <span class="nam">hf_model</span> <span class="op">=</span> <span class="nam">AutoModelForCausalLM</span><span class="op">.</span><span class="nam">from_pretrained</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1642" href="#t1642">1642</a></span><span class="t">                    <span class="nam">official_model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1643" href="#t1643">1643</a></span><span class="t">                    <span class="nam">revision</span><span class="op">=</span><span class="str">f"step{cfg.checkpoint_value}"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1644" href="#t1644">1644</a></span><span class="t">                    <span class="nam">torch_dtype</span><span class="op">=</span><span class="nam">dtype</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1645" href="#t1645">1645</a></span><span class="t">                    <span class="nam">token</span><span class="op">=</span><span class="nam">huggingface_token</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1646" href="#t1646">1646</a></span><span class="t">                    <span class="op">**</span><span class="nam">kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1647" href="#t1647">1647</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1648" href="#t1648">1648</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1649" href="#t1649">1649</a></span><span class="t">                <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span><span class="str">f"Checkpoints for model {official_model_name} are not supported"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1650" href="#t1650">1650</a></span><span class="t">        <span class="key">elif</span> <span class="nam">hf_model</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1650&#x202F;&#x219B;&#x202F;1678</span><span class="annotate long">line 1650 didn't jump to line 1678, because the condition on line 1650 was never false</span></span></p>
-    <p class="run"><span class="n"><a id="t1651" href="#t1651">1651</a></span><span class="t">            <span class="nam">huggingface_token</span> <span class="op">=</span> <span class="nam">os</span><span class="op">.</span><span class="nam">environ</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"HF_TOKEN"</span><span class="op">,</span> <span class="key">None</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1652" href="#t1652">1652</a></span><span class="t">            <span class="key">if</span> <span class="nam">official_model_name</span> <span class="key">in</span> <span class="nam">NON_HF_HOSTED_MODEL_NAMES</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1652&#x202F;&#x219B;&#x202F;1653</span><span class="annotate long">line 1652 didn't jump to line 1653, because the condition on line 1652 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1653" href="#t1653">1653</a></span><span class="t">                <span class="key">raise</span> <span class="nam">NotImplementedError</span><span class="op">(</span><span class="str">"Model not hosted on HuggingFace, must pass in hf_model"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1654" href="#t1654">1654</a></span><span class="t">            <span class="key">elif</span> <span class="str">"bert"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1655" href="#t1655">1655</a></span><span class="t">                <span class="nam">hf_model</span> <span class="op">=</span> <span class="nam">BertForPreTraining</span><span class="op">.</span><span class="nam">from_pretrained</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1656" href="#t1656">1656</a></span><span class="t">                    <span class="nam">official_model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1657" href="#t1657">1657</a></span><span class="t">                    <span class="nam">torch_dtype</span><span class="op">=</span><span class="nam">dtype</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1658" href="#t1658">1658</a></span><span class="t">                    <span class="nam">token</span><span class="op">=</span><span class="nam">huggingface_token</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1659" href="#t1659">1659</a></span><span class="t">                    <span class="op">**</span><span class="nam">kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1660" href="#t1660">1660</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1661" href="#t1661">1661</a></span><span class="t">            <span class="key">elif</span> <span class="str">"t5"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1662" href="#t1662">1662</a></span><span class="t">                <span class="nam">hf_model</span> <span class="op">=</span> <span class="nam">T5ForConditionalGeneration</span><span class="op">.</span><span class="nam">from_pretrained</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1663" href="#t1663">1663</a></span><span class="t">                    <span class="nam">official_model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1664" href="#t1664">1664</a></span><span class="t">                    <span class="nam">torch_dtype</span><span class="op">=</span><span class="nam">dtype</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1665" href="#t1665">1665</a></span><span class="t">                    <span class="nam">token</span><span class="op">=</span><span class="nam">huggingface_token</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1666" href="#t1666">1666</a></span><span class="t">                    <span class="op">**</span><span class="nam">kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1667" href="#t1667">1667</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1668" href="#t1668">1668</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1669" href="#t1669">1669</a></span><span class="t">                <span class="nam">hf_model</span> <span class="op">=</span> <span class="nam">AutoModelForCausalLM</span><span class="op">.</span><span class="nam">from_pretrained</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1670" href="#t1670">1670</a></span><span class="t">                    <span class="nam">official_model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1671" href="#t1671">1671</a></span><span class="t">                    <span class="nam">torch_dtype</span><span class="op">=</span><span class="nam">dtype</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1672" href="#t1672">1672</a></span><span class="t">                    <span class="nam">token</span><span class="op">=</span><span class="nam">huggingface_token</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1673" href="#t1673">1673</a></span><span class="t">                    <span class="op">**</span><span class="nam">kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1674" href="#t1674">1674</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1675" href="#t1675">1675</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1676" href="#t1676">1676</a></span><span class="t">            <span class="com"># Load model weights, and fold in layer norm weights</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1677" href="#t1677">1677</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1678" href="#t1678">1678</a></span><span class="t">        <span class="key">for</span> <span class="nam">param</span> <span class="key">in</span> <span class="nam">hf_model</span><span class="op">.</span><span class="nam">parameters</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1679" href="#t1679">1679</a></span><span class="t">            <span class="nam">param</span><span class="op">.</span><span class="nam">requires_grad</span> <span class="op">=</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1624" href="#t1624">1624</a></span><span class="t">        <span class="com"># Convert to dtype</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1625" href="#t1625">1625</a></span><span class="t">        <span class="nam">state_dict</span> <span class="op">=</span> <span class="op">{</span><span class="nam">k</span><span class="op">:</span> <span class="nam">v</span><span class="op">.</span><span class="nam">to</span><span class="op">(</span><span class="nam">dtype</span><span class="op">)</span> <span class="key">for</span> <span class="nam">k</span><span class="op">,</span> <span class="nam">v</span> <span class="key">in</span> <span class="nam">state_dict</span><span class="op">.</span><span class="nam">items</span><span class="op">(</span><span class="op">)</span><span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1626" href="#t1626">1626</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1627" href="#t1627">1627</a></span><span class="t">        <span class="key">if</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"neel-solu-old"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1628" href="#t1628">1628</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_neel_solu_old_weights</span><span class="op">(</span><span class="nam">state_dict</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1629" href="#t1629">1629</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"mingpt"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1630" href="#t1630">1630</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_mingpt_weights</span><span class="op">(</span><span class="nam">state_dict</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1631" href="#t1631">1631</a></span><span class="t">        <span class="key">return</span> <span class="nam">state_dict</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1632" href="#t1632">1632</a></span><span class="t">    <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1633" href="#t1633">1633</a></span><span class="t">        <span class="key">if</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">from_checkpoint</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1633&#x202F;&#x219B;&#x202F;1634</span><span class="annotate long">line 1633 didn't jump to line 1634, because the condition on line 1633 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1634" href="#t1634">1634</a></span><span class="t">            <span class="nam">huggingface_token</span> <span class="op">=</span> <span class="nam">os</span><span class="op">.</span><span class="nam">environ</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"HF_TOKEN"</span><span class="op">,</span> <span class="key">None</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1635" href="#t1635">1635</a></span><span class="t">            <span class="key">if</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"stanford-crfm"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1636" href="#t1636">1636</a></span><span class="t">                <span class="nam">hf_model</span> <span class="op">=</span> <span class="nam">AutoModelForCausalLM</span><span class="op">.</span><span class="nam">from_pretrained</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1637" href="#t1637">1637</a></span><span class="t">                    <span class="nam">official_model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1638" href="#t1638">1638</a></span><span class="t">                    <span class="nam">revision</span><span class="op">=</span><span class="str">f"checkpoint-{cfg.checkpoint_value}"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1639" href="#t1639">1639</a></span><span class="t">                    <span class="nam">torch_dtype</span><span class="op">=</span><span class="nam">dtype</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1640" href="#t1640">1640</a></span><span class="t">                    <span class="nam">token</span><span class="op">=</span><span class="nam">huggingface_token</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1641" href="#t1641">1641</a></span><span class="t">                    <span class="op">**</span><span class="nam">kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1642" href="#t1642">1642</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1643" href="#t1643">1643</a></span><span class="t">            <span class="key">elif</span> <span class="nam">official_model_name</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">"EleutherAI/pythia"</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1644" href="#t1644">1644</a></span><span class="t">                <span class="nam">hf_model</span> <span class="op">=</span> <span class="nam">AutoModelForCausalLM</span><span class="op">.</span><span class="nam">from_pretrained</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1645" href="#t1645">1645</a></span><span class="t">                    <span class="nam">official_model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1646" href="#t1646">1646</a></span><span class="t">                    <span class="nam">revision</span><span class="op">=</span><span class="str">f"step{cfg.checkpoint_value}"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1647" href="#t1647">1647</a></span><span class="t">                    <span class="nam">torch_dtype</span><span class="op">=</span><span class="nam">dtype</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1648" href="#t1648">1648</a></span><span class="t">                    <span class="nam">token</span><span class="op">=</span><span class="nam">huggingface_token</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1649" href="#t1649">1649</a></span><span class="t">                    <span class="op">**</span><span class="nam">kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1650" href="#t1650">1650</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1651" href="#t1651">1651</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1652" href="#t1652">1652</a></span><span class="t">                <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span><span class="str">f"Checkpoints for model {official_model_name} are not supported"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1653" href="#t1653">1653</a></span><span class="t">        <span class="key">elif</span> <span class="nam">hf_model</span> <span class="key">is</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1653&#x202F;&#x219B;&#x202F;1681</span><span class="annotate long">line 1653 didn't jump to line 1681, because the condition on line 1653 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t1654" href="#t1654">1654</a></span><span class="t">            <span class="nam">huggingface_token</span> <span class="op">=</span> <span class="nam">os</span><span class="op">.</span><span class="nam">environ</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">"HF_TOKEN"</span><span class="op">,</span> <span class="key">None</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1655" href="#t1655">1655</a></span><span class="t">            <span class="key">if</span> <span class="nam">official_model_name</span> <span class="key">in</span> <span class="nam">NON_HF_HOSTED_MODEL_NAMES</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1655&#x202F;&#x219B;&#x202F;1656</span><span class="annotate long">line 1655 didn't jump to line 1656, because the condition on line 1655 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1656" href="#t1656">1656</a></span><span class="t">                <span class="key">raise</span> <span class="nam">NotImplementedError</span><span class="op">(</span><span class="str">"Model not hosted on HuggingFace, must pass in hf_model"</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1657" href="#t1657">1657</a></span><span class="t">            <span class="key">elif</span> <span class="str">"bert"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1658" href="#t1658">1658</a></span><span class="t">                <span class="nam">hf_model</span> <span class="op">=</span> <span class="nam">BertForPreTraining</span><span class="op">.</span><span class="nam">from_pretrained</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1659" href="#t1659">1659</a></span><span class="t">                    <span class="nam">official_model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1660" href="#t1660">1660</a></span><span class="t">                    <span class="nam">torch_dtype</span><span class="op">=</span><span class="nam">dtype</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1661" href="#t1661">1661</a></span><span class="t">                    <span class="nam">token</span><span class="op">=</span><span class="nam">huggingface_token</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1662" href="#t1662">1662</a></span><span class="t">                    <span class="op">**</span><span class="nam">kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1663" href="#t1663">1663</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1664" href="#t1664">1664</a></span><span class="t">            <span class="key">elif</span> <span class="str">"t5"</span> <span class="key">in</span> <span class="nam">official_model_name</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1665" href="#t1665">1665</a></span><span class="t">                <span class="nam">hf_model</span> <span class="op">=</span> <span class="nam">T5ForConditionalGeneration</span><span class="op">.</span><span class="nam">from_pretrained</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1666" href="#t1666">1666</a></span><span class="t">                    <span class="nam">official_model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1667" href="#t1667">1667</a></span><span class="t">                    <span class="nam">torch_dtype</span><span class="op">=</span><span class="nam">dtype</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1668" href="#t1668">1668</a></span><span class="t">                    <span class="nam">token</span><span class="op">=</span><span class="nam">huggingface_token</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1669" href="#t1669">1669</a></span><span class="t">                    <span class="op">**</span><span class="nam">kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1670" href="#t1670">1670</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1671" href="#t1671">1671</a></span><span class="t">            <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1672" href="#t1672">1672</a></span><span class="t">                <span class="nam">hf_model</span> <span class="op">=</span> <span class="nam">AutoModelForCausalLM</span><span class="op">.</span><span class="nam">from_pretrained</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1673" href="#t1673">1673</a></span><span class="t">                    <span class="nam">official_model_name</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1674" href="#t1674">1674</a></span><span class="t">                    <span class="nam">torch_dtype</span><span class="op">=</span><span class="nam">dtype</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1675" href="#t1675">1675</a></span><span class="t">                    <span class="nam">token</span><span class="op">=</span><span class="nam">huggingface_token</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1676" href="#t1676">1676</a></span><span class="t">                    <span class="op">**</span><span class="nam">kwargs</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1677" href="#t1677">1677</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1678" href="#t1678">1678</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1679" href="#t1679">1679</a></span><span class="t">            <span class="com"># Load model weights, and fold in layer norm weights</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1680" href="#t1680">1680</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1681" href="#t1681">1681</a></span><span class="t">        <span class="key">if</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"GPT2LMHeadModel"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1682" href="#t1682">1682</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_gpt2_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1683" href="#t1683">1683</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"GPTNeoForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1684" href="#t1684">1684</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_neo_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1685" href="#t1685">1685</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"OPTForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1686" href="#t1686">1686</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_opt_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1687" href="#t1687">1687</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"GPTJForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1687&#x202F;&#x219B;&#x202F;1688</span><span class="annotate long">line 1687 didn't jump to line 1688, because the condition on line 1687 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1688" href="#t1688">1688</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_gptj_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1689" href="#t1689">1689</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"GPTNeoXForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1690" href="#t1690">1690</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_neox_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1691" href="#t1691">1691</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"LlamaForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1691&#x202F;&#x219B;&#x202F;1692</span><span class="annotate long">line 1691 didn't jump to line 1692, because the condition on line 1691 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1692" href="#t1692">1692</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_llama_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1693" href="#t1693">1693</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"BertForMaskedLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1694" href="#t1694">1694</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_bert_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1695" href="#t1695">1695</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"T5ForConditionalGeneration"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1696" href="#t1696">1696</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_t5_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1697" href="#t1697">1697</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"MistralForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1697&#x202F;&#x219B;&#x202F;1698</span><span class="annotate long">line 1697 didn't jump to line 1698, because the condition on line 1697 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1698" href="#t1698">1698</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_mistral_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1699" href="#t1699">1699</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"MixtralForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1699&#x202F;&#x219B;&#x202F;1700</span><span class="annotate long">line 1699 didn't jump to line 1700, because the condition on line 1699 was never true</span></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1700" href="#t1700">1700</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_mixtral_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1701" href="#t1701">1701</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"BloomForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1701&#x202F;&#x219B;&#x202F;1703</span><span class="annotate long">line 1701 didn't jump to line 1703, because the condition on line 1701 was never false</span></span></p>
-    <p class="run"><span class="n"><a id="t1702" href="#t1702">1702</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_bloom_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1703" href="#t1703">1703</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"GPT2LMHeadCustomModel"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1704" href="#t1704">1704</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_coder_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1705" href="#t1705">1705</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"QWenLMHeadModel"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1706" href="#t1706">1706</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_qwen_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1707" href="#t1707">1707</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"Qwen2ForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1708" href="#t1708">1708</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_qwen2_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1709" href="#t1709">1709</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"PhiForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1710" href="#t1710">1710</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_phi_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1711" href="#t1711">1711</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"Phi3ForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1712" href="#t1712">1712</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_phi3_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1713" href="#t1713">1713</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"GemmaForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1714" href="#t1714">1714</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_gemma_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1715" href="#t1715">1715</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"Gemma2ForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1716" href="#t1716">1716</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_gemma_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1717" href="#t1717">1717</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1718" href="#t1718">1718</a></span><span class="t">            <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1719" href="#t1719">1719</a></span><span class="t">                <span class="str">f"Loading weights from the architecture is not currently supported: {cfg.original_architecture}, generated from model name {cfg.model_name}. Feel free to open an issue on GitHub to request this feature."</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1720" href="#t1720">1720</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1721" href="#t1721">1721</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1722" href="#t1722">1722</a></span><span class="t">        <span class="key">return</span> <span class="nam">state_dict</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1723" href="#t1723">1723</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1681" href="#t1681">1681</a></span><span class="t">        <span class="key">for</span> <span class="nam">param</span> <span class="key">in</span> <span class="nam">hf_model</span><span class="op">.</span><span class="nam">parameters</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1682" href="#t1682">1682</a></span><span class="t">            <span class="nam">param</span><span class="op">.</span><span class="nam">requires_grad</span> <span class="op">=</span> <span class="key">False</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1683" href="#t1683">1683</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1684" href="#t1684">1684</a></span><span class="t">        <span class="key">if</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"GPT2LMHeadModel"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1685" href="#t1685">1685</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_gpt2_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1686" href="#t1686">1686</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"GPTNeoForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1687" href="#t1687">1687</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_neo_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1688" href="#t1688">1688</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"OPTForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1689" href="#t1689">1689</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_opt_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1690" href="#t1690">1690</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"GPTJForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1690&#x202F;&#x219B;&#x202F;1691</span><span class="annotate long">line 1690 didn't jump to line 1691, because the condition on line 1690 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1691" href="#t1691">1691</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_gptj_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1692" href="#t1692">1692</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"GPTNeoXForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1693" href="#t1693">1693</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_neox_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1694" href="#t1694">1694</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"LlamaForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1694&#x202F;&#x219B;&#x202F;1695</span><span class="annotate long">line 1694 didn't jump to line 1695, because the condition on line 1694 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1695" href="#t1695">1695</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_llama_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1696" href="#t1696">1696</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"BertForMaskedLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1697" href="#t1697">1697</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_bert_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1698" href="#t1698">1698</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"T5ForConditionalGeneration"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1699" href="#t1699">1699</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_t5_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1700" href="#t1700">1700</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"MistralForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1700&#x202F;&#x219B;&#x202F;1701</span><span class="annotate long">line 1700 didn't jump to line 1701, because the condition on line 1700 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1701" href="#t1701">1701</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_mistral_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1702" href="#t1702">1702</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"MixtralForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1702&#x202F;&#x219B;&#x202F;1703</span><span class="annotate long">line 1702 didn't jump to line 1703, because the condition on line 1702 was never true</span></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1703" href="#t1703">1703</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_mixtral_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1704" href="#t1704">1704</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"BloomForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1704&#x202F;&#x219B;&#x202F;1706</span><span class="annotate long">line 1704 didn't jump to line 1706, because the condition on line 1704 was never false</span></span></p>
+    <p class="run"><span class="n"><a id="t1705" href="#t1705">1705</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_bloom_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1706" href="#t1706">1706</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"GPT2LMHeadCustomModel"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1707" href="#t1707">1707</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_coder_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1708" href="#t1708">1708</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"QWenLMHeadModel"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1709" href="#t1709">1709</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_qwen_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1710" href="#t1710">1710</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"Qwen2ForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1711" href="#t1711">1711</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_qwen2_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1712" href="#t1712">1712</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"PhiForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1713" href="#t1713">1713</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_phi_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1714" href="#t1714">1714</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"Phi3ForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1715" href="#t1715">1715</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_phi3_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1716" href="#t1716">1716</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"GemmaForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1717" href="#t1717">1717</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_gemma_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1718" href="#t1718">1718</a></span><span class="t">        <span class="key">elif</span> <span class="nam">cfg</span><span class="op">.</span><span class="nam">original_architecture</span> <span class="op">==</span> <span class="str">"Gemma2ForCausalLM"</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1719" href="#t1719">1719</a></span><span class="t">            <span class="nam">state_dict</span> <span class="op">=</span> <span class="nam">convert_gemma_weights</span><span class="op">(</span><span class="nam">hf_model</span><span class="op">,</span> <span class="nam">cfg</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1720" href="#t1720">1720</a></span><span class="t">        <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1721" href="#t1721">1721</a></span><span class="t">            <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1722" href="#t1722">1722</a></span><span class="t">                <span class="str">f"Loading weights from the architecture is not currently supported: {cfg.original_architecture}, generated from model name {cfg.model_name}. Feel free to open an issue on GitHub to request this feature."</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1723" href="#t1723">1723</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1724" href="#t1724">1724</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1725" href="#t1725">1725</a></span><span class="t"><span class="key">def</span> <span class="nam">fill_missing_keys</span><span class="op">(</span><span class="nam">model</span><span class="op">,</span> <span class="nam">state_dict</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1726" href="#t1726">1726</a></span><span class="t">    <span class="str">"""Takes in a state dict from a pretrained model, and fills in any missing keys with the default initialization.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1725" href="#t1725">1725</a></span><span class="t">        <span class="key">return</span> <span class="nam">state_dict</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1726" href="#t1726">1726</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1727" href="#t1727">1727</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1728" href="#t1728">1728</a></span><span class="t"><span class="str">    This function is assumed to be run before weights are initialized.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1729" href="#t1729">1729</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1730" href="#t1730">1730</a></span><span class="t"><span class="str">    Args:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1731" href="#t1731">1731</a></span><span class="t"><span class="str">        state_dict (dict): State dict from a pretrained model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1728" href="#t1728">1728</a></span><span class="t"><span class="key">def</span> <span class="nam">fill_missing_keys</span><span class="op">(</span><span class="nam">model</span><span class="op">,</span> <span class="nam">state_dict</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1729" href="#t1729">1729</a></span><span class="t">    <span class="str">"""Takes in a state dict from a pretrained model, and fills in any missing keys with the default initialization.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1730" href="#t1730">1730</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1731" href="#t1731">1731</a></span><span class="t"><span class="str">    This function is assumed to be run before weights are initialized.</span>&nbsp;</span><span class="r"></span></p>
     <p class="pln"><span class="n"><a id="t1732" href="#t1732">1732</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1733" href="#t1733">1733</a></span><span class="t"><span class="str">    Returns:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1734" href="#t1734">1734</a></span><span class="t"><span class="str">        dict: State dict with missing keys filled in</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1735" href="#t1735">1735</a></span><span class="t"><span class="str">    """</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1736" href="#t1736">1736</a></span><span class="t">    <span class="com"># Get the default state dict</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1737" href="#t1737">1737</a></span><span class="t">    <span class="nam">default_state_dict</span> <span class="op">=</span> <span class="nam">model</span><span class="op">.</span><span class="nam">state_dict</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1738" href="#t1738">1738</a></span><span class="t">    <span class="com"># Get the keys that are missing from the pretrained model</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1739" href="#t1739">1739</a></span><span class="t">    <span class="nam">missing_keys</span> <span class="op">=</span> <span class="nam">set</span><span class="op">(</span><span class="nam">default_state_dict</span><span class="op">.</span><span class="nam">keys</span><span class="op">(</span><span class="op">)</span><span class="op">)</span> <span class="op">-</span> <span class="nam">set</span><span class="op">(</span><span class="nam">state_dict</span><span class="op">.</span><span class="nam">keys</span><span class="op">(</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1740" href="#t1740">1740</a></span><span class="t">    <span class="com"># Fill in the missing keys with the default initialization</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1741" href="#t1741">1741</a></span><span class="t">    <span class="key">for</span> <span class="nam">key</span> <span class="key">in</span> <span class="nam">missing_keys</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1742" href="#t1742">1742</a></span><span class="t">        <span class="key">if</span> <span class="str">"hf_model"</span> <span class="key">in</span> <span class="nam">key</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1742&#x202F;&#x219B;&#x202F;1744</span><span class="annotate long">line 1742 didn't jump to line 1744, because the condition on line 1742 was never true</span></span></p>
-    <p class="pln"><span class="n"><a id="t1743" href="#t1743">1743</a></span><span class="t">            <span class="com"># Skip keys that are from the HuggingFace model, if loading from HF.</span>&nbsp;</span><span class="r"></span></p>
-    <p class="mis show_mis"><span class="n"><a id="t1744" href="#t1744">1744</a></span><span class="t">            <span class="key">continue</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1745" href="#t1745">1745</a></span><span class="t">        <span class="key">if</span> <span class="str">"W_"</span> <span class="key">in</span> <span class="nam">key</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1746" href="#t1746">1746</a></span><span class="t">            <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1747" href="#t1747">1747</a></span><span class="t">                <span class="str">"Missing key for a weight matrix in pretrained, filled in with an empty tensor: {}"</span><span class="op">.</span><span class="nam">format</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1748" href="#t1748">1748</a></span><span class="t">                    <span class="nam">key</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1749" href="#t1749">1749</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1750" href="#t1750">1750</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1751" href="#t1751">1751</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">[</span><span class="nam">key</span><span class="op">]</span> <span class="op">=</span> <span class="nam">default_state_dict</span><span class="op">[</span><span class="nam">key</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1752" href="#t1752">1752</a></span><span class="t">    <span class="key">return</span> <span class="nam">state_dict</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1753" href="#t1753">1753</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1754" href="#t1754">1754</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="par run show_par"><span class="n"><a id="t1755" href="#t1755">1755</a></span><span class="t"><span class="op">@</span><span class="nam">dataclasses</span><span class="op">.</span><span class="nam">dataclass</span>&nbsp;</span><span class="r"><span class="annotate short">1755&#x202F;&#x219B;&#x202F;1757</span><span class="annotate long">line 1755 didn't jump to line 1757, because </span></span></p>
-    <p class="run"><span class="n"><a id="t1756" href="#t1756">1756</a></span><span class="t"><span class="key">class</span> <span class="nam">Config</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1757" href="#t1757">1757</a></span><span class="t">    <span class="nam">d_model</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">768</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1758" href="#t1758">1758</a></span><span class="t">    <span class="nam">debug</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1759" href="#t1759">1759</a></span><span class="t">    <span class="nam">layer_norm_eps</span><span class="op">:</span> <span class="nam">float</span> <span class="op">=</span> <span class="num">1e-5</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1760" href="#t1760">1760</a></span><span class="t">    <span class="nam">d_vocab</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">50257</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1761" href="#t1761">1761</a></span><span class="t">    <span class="nam">init_range</span><span class="op">:</span> <span class="nam">float</span> <span class="op">=</span> <span class="num">0.02</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1762" href="#t1762">1762</a></span><span class="t">    <span class="nam">n_ctx</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">1024</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1763" href="#t1763">1763</a></span><span class="t">    <span class="nam">d_head</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">64</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1764" href="#t1764">1764</a></span><span class="t">    <span class="nam">d_mlp</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">3072</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1765" href="#t1765">1765</a></span><span class="t">    <span class="nam">n_heads</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">12</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1766" href="#t1766">1766</a></span><span class="t">    <span class="nam">n_layers</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">12</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1767" href="#t1767">1767</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1768" href="#t1768">1768</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1769" href="#t1769">1769</a></span><span class="t"><span class="com"># Returns the configuration parameters of the model as a basic Config dataclass</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1770" href="#t1770">1770</a></span><span class="t"><span class="key">def</span> <span class="nam">get_basic_config</span><span class="op">(</span><span class="nam">model_name</span><span class="op">:</span> <span class="nam">str</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Config</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
-    <p class="run"><span class="n"><a id="t1771" href="#t1771">1771</a></span><span class="t">    <span class="key">return</span> <span class="nam">Config</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1772" href="#t1772">1772</a></span><span class="t">        <span class="op">**</span><span class="op">{</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1773" href="#t1773">1773</a></span><span class="t">            <span class="nam">k</span><span class="op">:</span> <span class="nam">v</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1774" href="#t1774">1774</a></span><span class="t">            <span class="key">for</span> <span class="nam">k</span><span class="op">,</span> <span class="nam">v</span> <span class="key">in</span> <span class="nam">get_pretrained_model_config</span><span class="op">(</span><span class="nam">model_name</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span><span class="op">.</span><span class="nam">to_dict</span><span class="op">(</span><span class="op">)</span><span class="op">.</span><span class="nam">items</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1775" href="#t1775">1775</a></span><span class="t">            <span class="key">if</span> <span class="nam">k</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1776" href="#t1776">1776</a></span><span class="t">            <span class="key">in</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1777" href="#t1777">1777</a></span><span class="t">                <span class="str">"d_model"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1778" href="#t1778">1778</a></span><span class="t">                <span class="str">"debug"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1779" href="#t1779">1779</a></span><span class="t">                <span class="str">"layer_norm_eps"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1780" href="#t1780">1780</a></span><span class="t">                <span class="str">"d_vocab"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1781" href="#t1781">1781</a></span><span class="t">                <span class="str">"init_range"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1782" href="#t1782">1782</a></span><span class="t">                <span class="str">"n_ctx"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1783" href="#t1783">1783</a></span><span class="t">                <span class="str">"d_head"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1784" href="#t1784">1784</a></span><span class="t">                <span class="str">"d_mlp"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1785" href="#t1785">1785</a></span><span class="t">                <span class="str">"n_heads"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1786" href="#t1786">1786</a></span><span class="t">                <span class="str">"n_layers"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1787" href="#t1787">1787</a></span><span class="t">            <span class="op">]</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1788" href="#t1788">1788</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
-    <p class="pln"><span class="n"><a id="t1789" href="#t1789">1789</a></span><span class="t">    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1733" href="#t1733">1733</a></span><span class="t"><span class="str">    Args:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1734" href="#t1734">1734</a></span><span class="t"><span class="str">        state_dict (dict): State dict from a pretrained model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1735" href="#t1735">1735</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1736" href="#t1736">1736</a></span><span class="t"><span class="str">    Returns:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1737" href="#t1737">1737</a></span><span class="t"><span class="str">        dict: State dict with missing keys filled in</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1738" href="#t1738">1738</a></span><span class="t"><span class="str">    """</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1739" href="#t1739">1739</a></span><span class="t">    <span class="com"># Get the default state dict</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1740" href="#t1740">1740</a></span><span class="t">    <span class="nam">default_state_dict</span> <span class="op">=</span> <span class="nam">model</span><span class="op">.</span><span class="nam">state_dict</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1741" href="#t1741">1741</a></span><span class="t">    <span class="com"># Get the keys that are missing from the pretrained model</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1742" href="#t1742">1742</a></span><span class="t">    <span class="nam">missing_keys</span> <span class="op">=</span> <span class="nam">set</span><span class="op">(</span><span class="nam">default_state_dict</span><span class="op">.</span><span class="nam">keys</span><span class="op">(</span><span class="op">)</span><span class="op">)</span> <span class="op">-</span> <span class="nam">set</span><span class="op">(</span><span class="nam">state_dict</span><span class="op">.</span><span class="nam">keys</span><span class="op">(</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1743" href="#t1743">1743</a></span><span class="t">    <span class="com"># Fill in the missing keys with the default initialization</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1744" href="#t1744">1744</a></span><span class="t">    <span class="key">for</span> <span class="nam">key</span> <span class="key">in</span> <span class="nam">missing_keys</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1745" href="#t1745">1745</a></span><span class="t">        <span class="key">if</span> <span class="str">"hf_model"</span> <span class="key">in</span> <span class="nam">key</span><span class="op">:</span>&nbsp;</span><span class="r"><span class="annotate short">1745&#x202F;&#x219B;&#x202F;1747</span><span class="annotate long">line 1745 didn't jump to line 1747, because the condition on line 1745 was never true</span></span></p>
+    <p class="pln"><span class="n"><a id="t1746" href="#t1746">1746</a></span><span class="t">            <span class="com"># Skip keys that are from the HuggingFace model, if loading from HF.</span>&nbsp;</span><span class="r"></span></p>
+    <p class="mis show_mis"><span class="n"><a id="t1747" href="#t1747">1747</a></span><span class="t">            <span class="key">continue</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1748" href="#t1748">1748</a></span><span class="t">        <span class="key">if</span> <span class="str">"W_"</span> <span class="key">in</span> <span class="nam">key</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1749" href="#t1749">1749</a></span><span class="t">            <span class="nam">logging</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1750" href="#t1750">1750</a></span><span class="t">                <span class="str">"Missing key for a weight matrix in pretrained, filled in with an empty tensor: {}"</span><span class="op">.</span><span class="nam">format</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1751" href="#t1751">1751</a></span><span class="t">                    <span class="nam">key</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1752" href="#t1752">1752</a></span><span class="t">                <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1753" href="#t1753">1753</a></span><span class="t">            <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1754" href="#t1754">1754</a></span><span class="t">        <span class="nam">state_dict</span><span class="op">[</span><span class="nam">key</span><span class="op">]</span> <span class="op">=</span> <span class="nam">default_state_dict</span><span class="op">[</span><span class="nam">key</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1755" href="#t1755">1755</a></span><span class="t">    <span class="key">return</span> <span class="nam">state_dict</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1756" href="#t1756">1756</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1757" href="#t1757">1757</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="par run show_par"><span class="n"><a id="t1758" href="#t1758">1758</a></span><span class="t"><span class="op">@</span><span class="nam">dataclasses</span><span class="op">.</span><span class="nam">dataclass</span>&nbsp;</span><span class="r"><span class="annotate short">1758&#x202F;&#x219B;&#x202F;1760</span><span class="annotate long">line 1758 didn't jump to line 1760, because </span></span></p>
+    <p class="run"><span class="n"><a id="t1759" href="#t1759">1759</a></span><span class="t"><span class="key">class</span> <span class="nam">Config</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1760" href="#t1760">1760</a></span><span class="t">    <span class="nam">d_model</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">768</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1761" href="#t1761">1761</a></span><span class="t">    <span class="nam">debug</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1762" href="#t1762">1762</a></span><span class="t">    <span class="nam">layer_norm_eps</span><span class="op">:</span> <span class="nam">float</span> <span class="op">=</span> <span class="num">1e-5</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1763" href="#t1763">1763</a></span><span class="t">    <span class="nam">d_vocab</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">50257</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1764" href="#t1764">1764</a></span><span class="t">    <span class="nam">init_range</span><span class="op">:</span> <span class="nam">float</span> <span class="op">=</span> <span class="num">0.02</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1765" href="#t1765">1765</a></span><span class="t">    <span class="nam">n_ctx</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">1024</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1766" href="#t1766">1766</a></span><span class="t">    <span class="nam">d_head</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">64</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1767" href="#t1767">1767</a></span><span class="t">    <span class="nam">d_mlp</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">3072</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1768" href="#t1768">1768</a></span><span class="t">    <span class="nam">n_heads</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">12</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1769" href="#t1769">1769</a></span><span class="t">    <span class="nam">n_layers</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">12</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1770" href="#t1770">1770</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1771" href="#t1771">1771</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1772" href="#t1772">1772</a></span><span class="t"><span class="com"># Returns the configuration parameters of the model as a basic Config dataclass</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1773" href="#t1773">1773</a></span><span class="t"><span class="key">def</span> <span class="nam">get_basic_config</span><span class="op">(</span><span class="nam">model_name</span><span class="op">:</span> <span class="nam">str</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span> <span class="op">-></span> <span class="nam">Config</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
+    <p class="run"><span class="n"><a id="t1774" href="#t1774">1774</a></span><span class="t">    <span class="key">return</span> <span class="nam">Config</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1775" href="#t1775">1775</a></span><span class="t">        <span class="op">**</span><span class="op">{</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1776" href="#t1776">1776</a></span><span class="t">            <span class="nam">k</span><span class="op">:</span> <span class="nam">v</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1777" href="#t1777">1777</a></span><span class="t">            <span class="key">for</span> <span class="nam">k</span><span class="op">,</span> <span class="nam">v</span> <span class="key">in</span> <span class="nam">get_pretrained_model_config</span><span class="op">(</span><span class="nam">model_name</span><span class="op">,</span> <span class="op">**</span><span class="nam">kwargs</span><span class="op">)</span><span class="op">.</span><span class="nam">to_dict</span><span class="op">(</span><span class="op">)</span><span class="op">.</span><span class="nam">items</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1778" href="#t1778">1778</a></span><span class="t">            <span class="key">if</span> <span class="nam">k</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1779" href="#t1779">1779</a></span><span class="t">            <span class="key">in</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1780" href="#t1780">1780</a></span><span class="t">                <span class="str">"d_model"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1781" href="#t1781">1781</a></span><span class="t">                <span class="str">"debug"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1782" href="#t1782">1782</a></span><span class="t">                <span class="str">"layer_norm_eps"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1783" href="#t1783">1783</a></span><span class="t">                <span class="str">"d_vocab"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1784" href="#t1784">1784</a></span><span class="t">                <span class="str">"init_range"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1785" href="#t1785">1785</a></span><span class="t">                <span class="str">"n_ctx"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1786" href="#t1786">1786</a></span><span class="t">                <span class="str">"d_head"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1787" href="#t1787">1787</a></span><span class="t">                <span class="str">"d_mlp"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1788" href="#t1788">1788</a></span><span class="t">                <span class="str">"n_heads"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1789" href="#t1789">1789</a></span><span class="t">                <span class="str">"n_layers"</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1790" href="#t1790">1790</a></span><span class="t">            <span class="op">]</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1791" href="#t1791">1791</a></span><span class="t">        <span class="op">}</span>&nbsp;</span><span class="r"></span></p>
+    <p class="pln"><span class="n"><a id="t1792" href="#t1792">1792</a></span><span class="t">    <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
 </main>
 <footer>
     <div class="content">
@@ -1880,7 +1883,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_past_key_value_caching_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_af97b5493da09a14_past_key_value_caching_py.html b/_static/coverage/d_af97b5493da09a14_past_key_value_caching_py.html
index ba9420d23..5051de2c9 100644
--- a/_static/coverage/d_af97b5493da09a14_past_key_value_caching_py.html
+++ b/_static/coverage/d_af97b5493da09a14_past_key_value_caching_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_patching_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -206,7 +206,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_patching_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_af97b5493da09a14_patching_py.html b/_static/coverage/d_af97b5493da09a14_patching_py.html
index 23317bff1..5fb3006be 100644
--- a/_static/coverage/d_af97b5493da09a14_patching_py.html
+++ b/_static/coverage/d_af97b5493da09a14_patching_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_712808f24eb400fe___init___py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -833,7 +833,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_712808f24eb400fe___init___py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_af97b5493da09a14_train_py.html b/_static/coverage/d_af97b5493da09a14_train_py.html
index 6a4f354c4..248302062 100644
--- a/_static/coverage/d_af97b5493da09a14_train_py.html
+++ b/_static/coverage/d_af97b5493da09a14_train_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_b2114f845e0399b7___init___py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -245,7 +245,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_b2114f845e0399b7___init___py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_af97b5493da09a14_utils_py.html b/_static/coverage/d_af97b5493da09a14_utils_py.html
index 3d8b19ad3..7d6dfc477 100644
--- a/_static/coverage/d_af97b5493da09a14_utils_py.html
+++ b/_static/coverage/d_af97b5493da09a14_utils_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="index.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -1328,7 +1328,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="index.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_b2114f845e0399b7___init___py.html b/_static/coverage/d_b2114f845e0399b7___init___py.html
index 8abf3f229..cacdc4702 100644
--- a/_static/coverage/d_b2114f845e0399b7___init___py.html
+++ b/_static/coverage/d_b2114f845e0399b7___init___py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_b2114f845e0399b7_activation_functions_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -91,7 +91,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_b2114f845e0399b7_activation_functions_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_b2114f845e0399b7_activation_functions_py.html b/_static/coverage/d_b2114f845e0399b7_activation_functions_py.html
index d7e30e549..99cc1e0ee 100644
--- a/_static/coverage/d_b2114f845e0399b7_activation_functions_py.html
+++ b/_static/coverage/d_b2114f845e0399b7_activation_functions_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_b2114f845e0399b7_addmm_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -117,7 +117,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_b2114f845e0399b7_addmm_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_b2114f845e0399b7_addmm_py.html b/_static/coverage/d_b2114f845e0399b7_addmm_py.html
index 91c0bef53..9e9110d28 100644
--- a/_static/coverage/d_b2114f845e0399b7_addmm_py.html
+++ b/_static/coverage/d_b2114f845e0399b7_addmm_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_b2114f845e0399b7_attention_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -126,7 +126,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_b2114f845e0399b7_attention_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_b2114f845e0399b7_attention_py.html b/_static/coverage/d_b2114f845e0399b7_attention_py.html
index 989872d8a..004a550ad 100644
--- a/_static/coverage/d_b2114f845e0399b7_attention_py.html
+++ b/_static/coverage/d_b2114f845e0399b7_attention_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_b2114f845e0399b7_devices_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -129,7 +129,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_b2114f845e0399b7_devices_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_b2114f845e0399b7_devices_py.html b/_static/coverage/d_b2114f845e0399b7_devices_py.html
index 2021cb29f..495d57281 100644
--- a/_static/coverage/d_b2114f845e0399b7_devices_py.html
+++ b/_static/coverage/d_b2114f845e0399b7_devices_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_utils_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -166,7 +166,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_utils_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7___init___py.html b/_static/coverage/d_c1ea89878f9b2ac7___init___py.html
index a7b6a3452..d753ac810 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7___init___py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7___init___py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_bert_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -111,7 +111,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_bert_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_bert_py.html b/_static/coverage/d_c1ea89878f9b2ac7_bert_py.html
index 9e300313f..cbf387fb3 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_bert_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_bert_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_bloom_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -156,7 +156,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_bloom_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_bloom_py.html b/_static/coverage/d_c1ea89878f9b2ac7_bloom_py.html
index b0ed9cf79..2cc0d76f7 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_bloom_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_bloom_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_coder_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -148,7 +148,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_coder_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_coder_py.html b/_static/coverage/d_c1ea89878f9b2ac7_coder_py.html
index 7ab8d1542..35555dc07 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_coder_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_coder_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_gemma_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -154,7 +154,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_gemma_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_gemma_py.html b/_static/coverage/d_c1ea89878f9b2ac7_gemma_py.html
index 566b08007..3da2c9dee 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_gemma_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_gemma_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_gpt2_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -186,7 +186,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_gpt2_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_gpt2_py.html b/_static/coverage/d_c1ea89878f9b2ac7_gpt2_py.html
index 23353d413..d795a0ffb 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_gpt2_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_gpt2_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_gptj_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -151,7 +151,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_gptj_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_gptj_py.html b/_static/coverage/d_c1ea89878f9b2ac7_gptj_py.html
index 80675c258..2be52a239 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_gptj_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_gptj_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_llama_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -141,7 +141,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_llama_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_llama_py.html b/_static/coverage/d_c1ea89878f9b2ac7_llama_py.html
index c7cd39f3c..607c45387 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_llama_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_llama_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_mingpt_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -187,7 +187,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_mingpt_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_mingpt_py.html b/_static/coverage/d_c1ea89878f9b2ac7_mingpt_py.html
index cb7d7658f..969246cd3 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_mingpt_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_mingpt_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_mistral_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -154,7 +154,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_mistral_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_mistral_py.html b/_static/coverage/d_c1ea89878f9b2ac7_mistral_py.html
index dabb9c320..97097a8ae 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_mistral_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_mistral_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_mixtral_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -148,7 +148,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_mixtral_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_mixtral_py.html b/_static/coverage/d_c1ea89878f9b2ac7_mixtral_py.html
index e2f842fe7..739eaa745 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_mixtral_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_mixtral_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_nanogpt_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -164,7 +164,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_nanogpt_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_nanogpt_py.html b/_static/coverage/d_c1ea89878f9b2ac7_nanogpt_py.html
index 0b95314d9..033b69238 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_nanogpt_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_nanogpt_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_neel_solu_old_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -179,7 +179,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_neel_solu_old_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_neel_solu_old_py.html b/_static/coverage/d_c1ea89878f9b2ac7_neel_solu_old_py.html
index d2fb1a1aa..bb07f262e 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_neel_solu_old_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_neel_solu_old_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_neo_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -129,7 +129,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_neo_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_neo_py.html b/_static/coverage/d_c1ea89878f9b2ac7_neo_py.html
index 518b56027..3cd8b022b 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_neo_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_neo_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_neox_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -140,7 +140,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_neox_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_neox_py.html b/_static/coverage/d_c1ea89878f9b2ac7_neox_py.html
index 7ad1abbc5..1648f586a 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_neox_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_neox_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_opt_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -150,7 +150,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_opt_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_opt_py.html b/_static/coverage/d_c1ea89878f9b2ac7_opt_py.html
index 9817e18a3..f0bea5cf3 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_opt_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_opt_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_phi_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -175,7 +175,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_phi_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_phi3_py.html b/_static/coverage/d_c1ea89878f9b2ac7_phi3_py.html
index 807aee63d..b49427298 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_phi3_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_phi3_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_qwen_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -147,7 +147,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_qwen_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_phi_py.html b/_static/coverage/d_c1ea89878f9b2ac7_phi_py.html
index c4eab8b28..f260c1b79 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_phi_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_phi_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_phi3_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -155,7 +155,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_phi3_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_qwen2_py.html b/_static/coverage/d_c1ea89878f9b2ac7_qwen2_py.html
index 24a67962e..7541cb569 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_qwen2_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_qwen2_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_t5_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -167,7 +167,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_t5_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_qwen_py.html b/_static/coverage/d_c1ea89878f9b2ac7_qwen_py.html
index 9d268eb6a..bef3d10cd 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_qwen_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_qwen_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_qwen2_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -156,7 +156,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_c1ea89878f9b2ac7_qwen2_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_c1ea89878f9b2ac7_t5_py.html b/_static/coverage/d_c1ea89878f9b2ac7_t5_py.html
index 947f94f29..4fa8ab53f 100644
--- a/_static/coverage/d_c1ea89878f9b2ac7_t5_py.html
+++ b/_static/coverage/d_c1ea89878f9b2ac7_t5_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_train_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -192,7 +192,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_train_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_db46118ef83ad831___init___py.html b/_static/coverage/d_db46118ef83ad831___init___py.html
index 8b51917ae..efe5106e3 100644
--- a/_static/coverage/d_db46118ef83ad831___init___py.html
+++ b/_static/coverage/d_db46118ef83ad831___init___py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_abstract_attention_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -123,7 +123,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_abstract_attention_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_db46118ef83ad831_abstract_attention_py.html b/_static/coverage/d_db46118ef83ad831_abstract_attention_py.html
index 48bf6ce3f..5a1890bae 100644
--- a/_static/coverage/d_db46118ef83ad831_abstract_attention_py.html
+++ b/_static/coverage/d_db46118ef83ad831_abstract_attention_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_attention_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -760,7 +760,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_attention_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_db46118ef83ad831_attention_py.html b/_static/coverage/d_db46118ef83ad831_attention_py.html
index 81cbb9feb..a2b7ebb2a 100644
--- a/_static/coverage/d_db46118ef83ad831_attention_py.html
+++ b/_static/coverage/d_db46118ef83ad831_attention_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_bert_block_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -150,7 +150,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_bert_block_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_db46118ef83ad831_bert_block_py.html b/_static/coverage/d_db46118ef83ad831_bert_block_py.html
index 7603af438..dbc4434b9 100644
--- a/_static/coverage/d_db46118ef83ad831_bert_block_py.html
+++ b/_static/coverage/d_db46118ef83ad831_bert_block_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_bert_embed_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -168,7 +168,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_bert_embed_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_db46118ef83ad831_bert_embed_py.html b/_static/coverage/d_db46118ef83ad831_bert_embed_py.html
index fcf817946..f01722778 100644
--- a/_static/coverage/d_db46118ef83ad831_bert_embed_py.html
+++ b/_static/coverage/d_db46118ef83ad831_bert_embed_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_bert_mlm_head_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -143,7 +143,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_bert_mlm_head_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_db46118ef83ad831_bert_mlm_head_py.html b/_static/coverage/d_db46118ef83ad831_bert_mlm_head_py.html
index ac530347e..f3ac87c39 100644
--- a/_static/coverage/d_db46118ef83ad831_bert_mlm_head_py.html
+++ b/_static/coverage/d_db46118ef83ad831_bert_mlm_head_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_embed_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -131,7 +131,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_embed_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_db46118ef83ad831_embed_py.html b/_static/coverage/d_db46118ef83ad831_embed_py.html
index ae489fd73..9d20e8f0e 100644
--- a/_static/coverage/d_db46118ef83ad831_embed_py.html
+++ b/_static/coverage/d_db46118ef83ad831_embed_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_grouped_query_attention_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -125,7 +125,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_grouped_query_attention_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_db46118ef83ad831_grouped_query_attention_py.html b/_static/coverage/d_db46118ef83ad831_grouped_query_attention_py.html
index 50183e9e4..5e9c23912 100644
--- a/_static/coverage/d_db46118ef83ad831_grouped_query_attention_py.html
+++ b/_static/coverage/d_db46118ef83ad831_grouped_query_attention_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_layer_norm_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -262,7 +262,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_layer_norm_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_db46118ef83ad831_layer_norm_pre_py.html b/_static/coverage/d_db46118ef83ad831_layer_norm_pre_py.html
index 87072b5b0..b29f8dda7 100644
--- a/_static/coverage/d_db46118ef83ad831_layer_norm_pre_py.html
+++ b/_static/coverage/d_db46118ef83ad831_layer_norm_pre_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_37285d613390727b_can_be_used_as_mlp_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -144,7 +144,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_37285d613390727b_can_be_used_as_mlp_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_db46118ef83ad831_layer_norm_py.html b/_static/coverage/d_db46118ef83ad831_layer_norm_py.html
index c889ab252..69d768d82 100644
--- a/_static/coverage/d_db46118ef83ad831_layer_norm_py.html
+++ b/_static/coverage/d_db46118ef83ad831_layer_norm_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_layer_norm_pre_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -147,7 +147,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_layer_norm_pre_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_db46118ef83ad831_pos_embed_py.html b/_static/coverage/d_db46118ef83ad831_pos_embed_py.html
index 4a33e6388..b07f45c19 100644
--- a/_static/coverage/d_db46118ef83ad831_pos_embed_py.html
+++ b/_static/coverage/d_db46118ef83ad831_pos_embed_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_rms_norm_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -160,7 +160,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_rms_norm_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_db46118ef83ad831_rms_norm_pre_py.html b/_static/coverage/d_db46118ef83ad831_rms_norm_pre_py.html
index 517e6ec10..274419321 100644
--- a/_static/coverage/d_db46118ef83ad831_rms_norm_pre_py.html
+++ b/_static/coverage/d_db46118ef83ad831_rms_norm_pre_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_t5_attention_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -126,7 +126,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_t5_attention_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_db46118ef83ad831_rms_norm_py.html b/_static/coverage/d_db46118ef83ad831_rms_norm_py.html
index f670c3f14..fbaeeea4c 100644
--- a/_static/coverage/d_db46118ef83ad831_rms_norm_py.html
+++ b/_static/coverage/d_db46118ef83ad831_rms_norm_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_rms_norm_pre_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -136,7 +136,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_rms_norm_pre_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_db46118ef83ad831_t5_attention_py.html b/_static/coverage/d_db46118ef83ad831_t5_attention_py.html
index b62b6eefe..fe3d53a05 100644
--- a/_static/coverage/d_db46118ef83ad831_t5_attention_py.html
+++ b/_static/coverage/d_db46118ef83ad831_t5_attention_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_t5_block_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -231,7 +231,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_t5_block_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_db46118ef83ad831_t5_block_py.html b/_static/coverage/d_db46118ef83ad831_t5_block_py.html
index 057501664..2fdd089f7 100644
--- a/_static/coverage/d_db46118ef83ad831_t5_block_py.html
+++ b/_static/coverage/d_db46118ef83ad831_t5_block_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_token_typed_embed_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -247,7 +247,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_token_typed_embed_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_db46118ef83ad831_token_typed_embed_py.html b/_static/coverage/d_db46118ef83ad831_token_typed_embed_py.html
index afb463dc6..223dda3b8 100644
--- a/_static/coverage/d_db46118ef83ad831_token_typed_embed_py.html
+++ b/_static/coverage/d_db46118ef83ad831_token_typed_embed_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_transformer_block_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -118,7 +118,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_transformer_block_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_db46118ef83ad831_transformer_block_py.html b/_static/coverage/d_db46118ef83ad831_transformer_block_py.html
index 3d7ffc660..e9a6e88a1 100644
--- a/_static/coverage/d_db46118ef83ad831_transformer_block_py.html
+++ b/_static/coverage/d_db46118ef83ad831_transformer_block_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_unembed_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -299,7 +299,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_db46118ef83ad831_unembed_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/d_db46118ef83ad831_unembed_py.html b/_static/coverage/d_db46118ef83ad831_unembed_py.html
index 340254d3f..b73dce1b8 100644
--- a/_static/coverage/d_db46118ef83ad831_unembed_py.html
+++ b/_static/coverage/d_db46118ef83ad831_unembed_py.html
@@ -67,7 +67,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_evals_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"/>
@@ -122,7 +122,7 @@ <h2>
             <a id="nextFileLink" class="nav" href="d_af97b5493da09a14_evals_py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </footer>
diff --git a/_static/coverage/index.html b/_static/coverage/index.html
index 2b8df113d..216551536 100644
--- a/_static/coverage/index.html
+++ b/_static/coverage/index.html
@@ -47,7 +47,7 @@ <h1>Coverage report:
         </form>
         <p class="text">
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
 </header>
@@ -391,12 +391,12 @@ <h1>Coverage report:
             </tr>
             <tr class="file">
                 <td class="name left"><a href="d_af97b5493da09a14_loading_from_pretrained_py.html">transformer_lens/loading_from_pretrained.py</a></td>
-                <td>306</td>
-                <td>106</td>
+                <td>308</td>
+                <td>107</td>
                 <td>0</td>
-                <td>202</td>
-                <td>46</td>
-                <td class="right" data-ratio="314 508">62%</td>
+                <td>204</td>
+                <td>47</td>
+                <td class="right" data-ratio="316 512">62%</td>
             </tr>
             <tr class="file">
                 <td class="name left"><a href="d_af97b5493da09a14_past_key_value_caching_py.html">transformer_lens/past_key_value_caching.py</a></td>
@@ -681,12 +681,12 @@ <h1>Coverage report:
         <tfoot>
             <tr class="total">
                 <td class="name left">Total</td>
-                <td>4964</td>
-                <td>1153</td>
+                <td>4966</td>
+                <td>1154</td>
                 <td>0</td>
-                <td>1940</td>
-                <td>316</td>
-                <td class="right" data-ratio="5067 6904">73%</td>
+                <td>1942</td>
+                <td>317</td>
+                <td class="right" data-ratio="5069 6908">73%</td>
             </tr>
         </tfoot>
     </table>
@@ -698,7 +698,7 @@ <h1>Coverage report:
     <div class="content">
         <p>
             <a class="nav" href="https://coverage.readthedocs.io/en/7.4.4">coverage.py v7.4.4</a>,
-            created at 2024-09-05 17:34 +0000
+            created at 2024-09-10 17:09 +0000
         </p>
     </div>
     <aside class="hidden">
diff --git a/_static/coverage/status.json b/_static/coverage/status.json
index 5c01a5cf8..59292b5b8 100644
--- a/_static/coverage/status.json
+++ b/_static/coverage/status.json
@@ -1 +1 @@
-{"note":"This file is an internal implementation detail to speed up HTML report generation. Its format can change at any time. You might be looking for the JSON report: https://coverage.rtfd.io/cmd.html#cmd-json","format":2,"version":"7.4.4","globals":"f7ec83bc01aec1de67db4e1d62b20997","files":{"d_af97b5493da09a14_ActivationCache_py":{"hash":"9e688d29b5d7e756b245666804e5f24e","index":{"nums":[0,1,288,0,7,168,10,16],"html_filename":"d_af97b5493da09a14_ActivationCache_py.html","relative_filename":"transformer_lens/ActivationCache.py"}},"d_af97b5493da09a14_FactoredMatrix_py":{"hash":"3731c82dd9da1b8aa7d998cf8cf5db55","index":{"nums":[0,1,129,0,2,60,3,5],"html_filename":"d_af97b5493da09a14_FactoredMatrix_py.html","relative_filename":"transformer_lens/FactoredMatrix.py"}},"d_af97b5493da09a14_HookedEncoder_py":{"hash":"8b1980ed46804b90fefe423d7fdee191","index":{"nums":[0,1,162,0,10,104,33,35],"html_filename":"d_af97b5493da09a14_HookedEncoder_py.html","relative_filename":"transformer_lens/HookedEncoder.py"}},"d_af97b5493da09a14_HookedEncoderDecoder_py":{"hash":"ced41e685cd841a10cfabecef98df15c","index":{"nums":[0,1,170,0,19,112,41,45],"html_filename":"d_af97b5493da09a14_HookedEncoderDecoder_py.html","relative_filename":"transformer_lens/HookedEncoderDecoder.py"}},"d_af97b5493da09a14_HookedTransformer_py":{"hash":"8c58328645a8f84166c3b82176fa9167","index":{"nums":[0,1,733,0,182,454,77,177],"html_filename":"d_af97b5493da09a14_HookedTransformer_py.html","relative_filename":"transformer_lens/HookedTransformer.py"}},"d_af97b5493da09a14_HookedTransformerConfig_py":{"hash":"d93dc6fb35f61724ec3e37160f10353a","index":{"nums":[0,1,130,0,8,40,7,7],"html_filename":"d_af97b5493da09a14_HookedTransformerConfig_py.html","relative_filename":"transformer_lens/HookedTransformerConfig.py"}},"d_af97b5493da09a14_SVDInterpreter_py":{"hash":"8b9a2792be15afabc033f624683fed44","index":{"nums":[0,1,55,0,1,16,1,1],"html_filename":"d_af97b5493da09a14_SVDInterpreter_py.html","relative_filename":"transformer_lens/SVDInterpreter.py"}},"d_af97b5493da09a14___init___py":{"hash":"b5c172d6f5d0c163dd8d3d8a188ef8fc","index":{"nums":[0,1,20,0,0,0,0,0],"html_filename":"d_af97b5493da09a14___init___py.html","relative_filename":"transformer_lens/__init__.py"}},"d_db46118ef83ad831___init___py":{"hash":"0f5ac43371106089cb8859730506eb31","index":{"nums":[0,1,20,0,0,0,0,0],"html_filename":"d_db46118ef83ad831___init___py.html","relative_filename":"transformer_lens/components/__init__.py"}},"d_db46118ef83ad831_abstract_attention_py":{"hash":"1e34e6bd63d5f4553e7a154f5619a95d","index":{"nums":[0,1,220,0,27,84,18,22],"html_filename":"d_db46118ef83ad831_abstract_attention_py.html","relative_filename":"transformer_lens/components/abstract_attention.py"}},"d_db46118ef83ad831_attention_py":{"hash":"771ce2a06e2204a8ea0b5c09549b0faa","index":{"nums":[0,1,20,0,4,4,2,2],"html_filename":"d_db46118ef83ad831_attention_py.html","relative_filename":"transformer_lens/components/attention.py"}},"d_db46118ef83ad831_bert_block_py":{"hash":"1d3faf4dfb51c82ef0e70e027c4442f7","index":{"nums":[0,1,45,0,4,2,1,1],"html_filename":"d_db46118ef83ad831_bert_block_py.html","relative_filename":"transformer_lens/components/bert_block.py"}},"d_db46118ef83ad831_bert_embed_py":{"hash":"fbb9b2fc553abf0e4d8a0afa34f58817","index":{"nums":[0,1,30,0,0,2,0,0],"html_filename":"d_db46118ef83ad831_bert_embed_py.html","relative_filename":"transformer_lens/components/bert_embed.py"}},"d_db46118ef83ad831_bert_mlm_head_py":{"hash":"0c5ab7d0820de68c654488b524f45c4d","index":{"nums":[0,1,20,0,0,0,0,0],"html_filename":"d_db46118ef83ad831_bert_mlm_head_py.html","relative_filename":"transformer_lens/components/bert_mlm_head.py"}},"d_db46118ef83ad831_embed_py":{"hash":"e80917c423b62b1fffeee5455ec2d952","index":{"nums":[0,1,17,0,0,4,0,0],"html_filename":"d_db46118ef83ad831_embed_py.html","relative_filename":"transformer_lens/components/embed.py"}},"d_db46118ef83ad831_grouped_query_attention_py":{"hash":"81ede5e95b90ea52c9793e25f7291b48","index":{"nums":[0,1,53,0,4,16,0,0],"html_filename":"d_db46118ef83ad831_grouped_query_attention_py.html","relative_filename":"transformer_lens/components/grouped_query_attention.py"}},"d_db46118ef83ad831_layer_norm_py":{"hash":"5d34676f9227c3c3bd83d13ca4a762ec","index":{"nums":[0,1,25,0,1,4,1,1],"html_filename":"d_db46118ef83ad831_layer_norm_py.html","relative_filename":"transformer_lens/components/layer_norm.py"}},"d_db46118ef83ad831_layer_norm_pre_py":{"hash":"a488e1ab3adf495070bf5e03582eee99","index":{"nums":[0,1,19,0,1,2,1,1],"html_filename":"d_db46118ef83ad831_layer_norm_pre_py.html","relative_filename":"transformer_lens/components/layer_norm_pre.py"}},"d_37285d613390727b_can_be_used_as_mlp_py":{"hash":"199b3e1e1d8fc5533d79c3788e257832","index":{"nums":[0,1,30,0,1,6,1,1],"html_filename":"d_37285d613390727b_can_be_used_as_mlp_py.html","relative_filename":"transformer_lens/components/mlps/can_be_used_as_mlp.py"}},"d_37285d613390727b_gated_mlp_py":{"hash":"1b9d23e893279e5255ac14d69e026485","index":{"nums":[0,1,31,0,3,4,2,2],"html_filename":"d_37285d613390727b_gated_mlp_py.html","relative_filename":"transformer_lens/components/mlps/gated_mlp.py"}},"d_37285d613390727b_gated_mlp_4bit_py":{"hash":"e85f0742f0a4142be44d967799763a15","index":{"nums":[0,1,32,0,18,4,1,3],"html_filename":"d_37285d613390727b_gated_mlp_4bit_py.html","relative_filename":"transformer_lens/components/mlps/gated_mlp_4bit.py"}},"d_37285d613390727b_mlp_py":{"hash":"a3901498f90188ad75b2b640144e7070","index":{"nums":[0,1,25,0,0,2,0,0],"html_filename":"d_37285d613390727b_mlp_py.html","relative_filename":"transformer_lens/components/mlps/mlp.py"}},"d_37285d613390727b_moe_py":{"hash":"c69707a0407a7cacf0929fa3233216f9","index":{"nums":[0,1,59,0,1,6,1,1],"html_filename":"d_37285d613390727b_moe_py.html","relative_filename":"transformer_lens/components/mlps/moe.py"}},"d_db46118ef83ad831_pos_embed_py":{"hash":"3688508c2257f2f12a9e431f92fdcd2a","index":{"nums":[0,1,23,0,0,2,0,0],"html_filename":"d_db46118ef83ad831_pos_embed_py.html","relative_filename":"transformer_lens/components/pos_embed.py"}},"d_db46118ef83ad831_rms_norm_py":{"hash":"19408e8189b8bb5d095ae6bb05045ef1","index":{"nums":[0,1,23,0,2,4,2,2],"html_filename":"d_db46118ef83ad831_rms_norm_py.html","relative_filename":"transformer_lens/components/rms_norm.py"}},"d_db46118ef83ad831_rms_norm_pre_py":{"hash":"04bea994792415493e41118c6e6c4da2","index":{"nums":[0,1,18,0,1,2,1,1],"html_filename":"d_db46118ef83ad831_rms_norm_pre_py.html","relative_filename":"transformer_lens/components/rms_norm_pre.py"}},"d_db46118ef83ad831_t5_attention_py":{"hash":"b02b9250eaf1c84e129e8763676c2d48","index":{"nums":[0,1,50,0,3,12,3,3],"html_filename":"d_db46118ef83ad831_t5_attention_py.html","relative_filename":"transformer_lens/components/t5_attention.py"}},"d_db46118ef83ad831_t5_block_py":{"hash":"8efa9a61311e44c6263d8dd61588fac4","index":{"nums":[0,1,64,0,6,12,3,3],"html_filename":"d_db46118ef83ad831_t5_block_py.html","relative_filename":"transformer_lens/components/t5_block.py"}},"d_db46118ef83ad831_token_typed_embed_py":{"hash":"0265390bcc2c91e19e04aee0cc722a2b","index":{"nums":[0,1,12,0,0,0,0,0],"html_filename":"d_db46118ef83ad831_token_typed_embed_py.html","relative_filename":"transformer_lens/components/token_typed_embed.py"}},"d_db46118ef83ad831_transformer_block_py":{"hash":"8fa97e0fe78b9c672714f56402884a44","index":{"nums":[0,1,101,0,15,52,10,18],"html_filename":"d_db46118ef83ad831_transformer_block_py.html","relative_filename":"transformer_lens/components/transformer_block.py"}},"d_db46118ef83ad831_unembed_py":{"hash":"a662f40d2a2543805f0b9f0d06f5b5c4","index":{"nums":[0,1,14,0,0,0,0,0],"html_filename":"d_db46118ef83ad831_unembed_py.html","relative_filename":"transformer_lens/components/unembed.py"}},"d_af97b5493da09a14_evals_py":{"hash":"7d496aae736b2525c13fe25d5363328b","index":{"nums":[0,1,147,0,50,46,2,14],"html_filename":"d_af97b5493da09a14_evals_py.html","relative_filename":"transformer_lens/evals.py"}},"d_65d4430f90bfb219_activation_function_factory_py":{"hash":"c4701f74bdab03c4b2cf3f7cd8afd980","index":{"nums":[0,1,12,0,2,6,2,2],"html_filename":"d_65d4430f90bfb219_activation_function_factory_py.html","relative_filename":"transformer_lens/factories/activation_function_factory.py"}},"d_65d4430f90bfb219_mlp_factory_py":{"hash":"4822436b1bacb210c29f36d16e51b216","index":{"nums":[0,1,14,0,1,6,1,1],"html_filename":"d_65d4430f90bfb219_mlp_factory_py.html","relative_filename":"transformer_lens/factories/mlp_factory.py"}},"d_af97b5493da09a14_head_detector_py":{"hash":"373e4b6b6945f7eec6de3b6987c4a9de","index":{"nums":[0,1,84,0,2,32,2,2],"html_filename":"d_af97b5493da09a14_head_detector_py.html","relative_filename":"transformer_lens/head_detector.py"}},"d_af97b5493da09a14_hook_points_py":{"hash":"cb760187bbf7a69669c316b56e7f9b12","index":{"nums":[0,1,234,0,41,124,19,49],"html_filename":"d_af97b5493da09a14_hook_points_py.html","relative_filename":"transformer_lens/hook_points.py"}},"d_af97b5493da09a14_loading_from_pretrained_py":{"hash":"adb7145cc9fd3384d1aa585cfe387c86","index":{"nums":[0,1,306,0,106,202,46,88],"html_filename":"d_af97b5493da09a14_loading_from_pretrained_py.html","relative_filename":"transformer_lens/loading_from_pretrained.py"}},"d_af97b5493da09a14_past_key_value_caching_py":{"hash":"994a13780a7f92b6a2ab53a37e662c15","index":{"nums":[0,1,46,0,0,18,2,2],"html_filename":"d_af97b5493da09a14_past_key_value_caching_py.html","relative_filename":"transformer_lens/past_key_value_caching.py"}},"d_af97b5493da09a14_patching_py":{"hash":"d96c96e4ecf5fcfe45d2838d27a9d918","index":{"nums":[0,1,140,0,70,18,0,14],"html_filename":"d_af97b5493da09a14_patching_py.html","relative_filename":"transformer_lens/patching.py"}},"d_712808f24eb400fe___init___py":{"hash":"ca2411bb2d5283cba14da3ce60f5feec","index":{"nums":[0,1,0,0,0,0,0,0],"html_filename":"d_712808f24eb400fe___init___py.html","relative_filename":"transformer_lens/pretrained/__init__.py"}},"d_c1ea89878f9b2ac7___init___py":{"hash":"2994f6b388ff62b0d84269971b0766e3","index":{"nums":[0,1,20,0,0,0,0,0],"html_filename":"d_c1ea89878f9b2ac7___init___py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/__init__.py"}},"d_c1ea89878f9b2ac7_bert_py":{"hash":"1de2f9e5bcd80d6bcc6794f7073bbe66","index":{"nums":[0,1,31,0,0,2,0,0],"html_filename":"d_c1ea89878f9b2ac7_bert_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/bert.py"}},"d_c1ea89878f9b2ac7_bloom_py":{"hash":"28eee61cd9581bbb556c20abd44717d1","index":{"nums":[0,1,40,0,0,2,0,0],"html_filename":"d_c1ea89878f9b2ac7_bloom_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/bloom.py"}},"d_c1ea89878f9b2ac7_coder_py":{"hash":"008f8c07ead34001e59d834e9672b208","index":{"nums":[0,1,43,0,38,2,0,2],"html_filename":"d_c1ea89878f9b2ac7_coder_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/coder.py"}},"d_c1ea89878f9b2ac7_gemma_py":{"hash":"b3ee2563edaf1aec4f1db3777cf34c84","index":{"nums":[0,1,41,0,36,6,0,6],"html_filename":"d_c1ea89878f9b2ac7_gemma_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/gemma.py"}},"d_c1ea89878f9b2ac7_gpt2_py":{"hash":"b6864d10cf3b70f76be00e47695dbec2","index":{"nums":[0,1,39,0,0,2,0,0],"html_filename":"d_c1ea89878f9b2ac7_gpt2_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/gpt2.py"}},"d_c1ea89878f9b2ac7_gptj_py":{"hash":"6b6d95ba6133fe6935f33c2431cec376","index":{"nums":[0,1,36,0,31,2,0,2],"html_filename":"d_c1ea89878f9b2ac7_gptj_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/gptj.py"}},"d_c1ea89878f9b2ac7_llama_py":{"hash":"b283de32d58e72f23d1dde8b2a54f278","index":{"nums":[0,1,45,0,39,8,0,8],"html_filename":"d_c1ea89878f9b2ac7_llama_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/llama.py"}},"d_c1ea89878f9b2ac7_mingpt_py":{"hash":"8afdb1ccd308a76b7af9ea71404ed112","index":{"nums":[0,1,40,0,0,2,0,0],"html_filename":"d_c1ea89878f9b2ac7_mingpt_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/mingpt.py"}},"d_c1ea89878f9b2ac7_mistral_py":{"hash":"f08ee3cdaa7c2dfb152ef82a4917497f","index":{"nums":[0,1,36,0,31,2,0,2],"html_filename":"d_c1ea89878f9b2ac7_mistral_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/mistral.py"}},"d_c1ea89878f9b2ac7_mixtral_py":{"hash":"bb26007d26637b02b3869d4733d0d6e8","index":{"nums":[0,1,37,0,32,4,0,4],"html_filename":"d_c1ea89878f9b2ac7_mixtral_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/mixtral.py"}},"d_c1ea89878f9b2ac7_nanogpt_py":{"hash":"e2ff3818963256534c9ca6a4ff6566a5","index":{"nums":[0,1,52,0,47,10,0,10],"html_filename":"d_c1ea89878f9b2ac7_nanogpt_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/nanogpt.py"}},"d_c1ea89878f9b2ac7_neel_solu_old_py":{"hash":"d96be68dd2fba5dbcba015e7f8b10fc1","index":{"nums":[0,1,17,0,0,12,2,2],"html_filename":"d_c1ea89878f9b2ac7_neel_solu_old_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/neel_solu_old.py"}},"d_c1ea89878f9b2ac7_neo_py":{"hash":"67f2d6bb1adda03d8d3eacbe5e1a0cb4","index":{"nums":[0,1,37,0,0,2,0,0],"html_filename":"d_c1ea89878f9b2ac7_neo_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/neo.py"}},"d_c1ea89878f9b2ac7_neox_py":{"hash":"fde01ee4309dd58bf3a6793ca3421f9e","index":{"nums":[0,1,34,0,0,2,0,0],"html_filename":"d_c1ea89878f9b2ac7_neox_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/neox.py"}},"d_c1ea89878f9b2ac7_opt_py":{"hash":"5bd938981ffdf863cddf1e54a3526615","index":{"nums":[0,1,40,0,0,2,0,0],"html_filename":"d_c1ea89878f9b2ac7_opt_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/opt.py"}},"d_c1ea89878f9b2ac7_phi_py":{"hash":"487a556d6755479e53397cfa39646490","index":{"nums":[0,1,41,0,37,2,0,2],"html_filename":"d_c1ea89878f9b2ac7_phi_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/phi.py"}},"d_c1ea89878f9b2ac7_phi3_py":{"hash":"9d833a74a1082fc28aa204b928511d91","index":{"nums":[0,1,35,0,30,2,0,2],"html_filename":"d_c1ea89878f9b2ac7_phi3_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/phi3.py"}},"d_c1ea89878f9b2ac7_qwen_py":{"hash":"5abcfa29a1d0456fb304a4b3b7ee2063","index":{"nums":[0,1,38,0,33,2,0,2],"html_filename":"d_c1ea89878f9b2ac7_qwen_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/qwen.py"}},"d_c1ea89878f9b2ac7_qwen2_py":{"hash":"4f2b989be42046944eb76e787f12f8a4","index":{"nums":[0,1,41,0,36,2,0,2],"html_filename":"d_c1ea89878f9b2ac7_qwen2_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/qwen2.py"}},"d_c1ea89878f9b2ac7_t5_py":{"hash":"fb47f0cb69cfc6778072b400fe6e53ef","index":{"nums":[0,1,33,0,0,4,0,0],"html_filename":"d_c1ea89878f9b2ac7_t5_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/t5.py"}},"d_af97b5493da09a14_train_py":{"hash":"74c4dc0140e6c628f623287b84708fb2","index":{"nums":[0,1,72,0,42,34,1,33],"html_filename":"d_af97b5493da09a14_train_py.html","relative_filename":"transformer_lens/train.py"}},"d_b2114f845e0399b7___init___py":{"hash":"ca2411bb2d5283cba14da3ce60f5feec","index":{"nums":[0,1,0,0,0,0,0,0],"html_filename":"d_b2114f845e0399b7___init___py.html","relative_filename":"transformer_lens/utilities/__init__.py"}},"d_b2114f845e0399b7_activation_functions_py":{"hash":"0926a004863506fd686e3d1e0ae5eb84","index":{"nums":[0,1,6,0,0,2,0,0],"html_filename":"d_b2114f845e0399b7_activation_functions_py.html","relative_filename":"transformer_lens/utilities/activation_functions.py"}},"d_b2114f845e0399b7_addmm_py":{"hash":"b1cb8300653ca4c3dd6a4f5c06408a84","index":{"nums":[0,1,10,0,0,0,0,0],"html_filename":"d_b2114f845e0399b7_addmm_py.html","relative_filename":"transformer_lens/utilities/addmm.py"}},"d_b2114f845e0399b7_attention_py":{"hash":"7f944a3a3a9cf8f8df0018d09fea2fae","index":{"nums":[0,1,10,0,0,0,0,0],"html_filename":"d_b2114f845e0399b7_attention_py.html","relative_filename":"transformer_lens/utilities/attention.py"}},"d_b2114f845e0399b7_devices_py":{"hash":"1da1c1a01ae3a7697bbb04016881df11","index":{"nums":[0,1,31,0,7,18,4,10],"html_filename":"d_b2114f845e0399b7_devices_py.html","relative_filename":"transformer_lens/utilities/devices.py"}},"d_af97b5493da09a14_utils_py":{"hash":"6f14e8d0ec39c4d104978baee513b6e6","index":{"nums":[0,1,438,0,122,184,16,78],"html_filename":"d_af97b5493da09a14_utils_py.html","relative_filename":"transformer_lens/utils.py"}}}}
\ No newline at end of file
+{"note":"This file is an internal implementation detail to speed up HTML report generation. Its format can change at any time. You might be looking for the JSON report: https://coverage.rtfd.io/cmd.html#cmd-json","format":2,"version":"7.4.4","globals":"f7ec83bc01aec1de67db4e1d62b20997","files":{"d_af97b5493da09a14_ActivationCache_py":{"hash":"9e688d29b5d7e756b245666804e5f24e","index":{"nums":[0,1,288,0,7,168,10,16],"html_filename":"d_af97b5493da09a14_ActivationCache_py.html","relative_filename":"transformer_lens/ActivationCache.py"}},"d_af97b5493da09a14_FactoredMatrix_py":{"hash":"3731c82dd9da1b8aa7d998cf8cf5db55","index":{"nums":[0,1,129,0,2,60,3,5],"html_filename":"d_af97b5493da09a14_FactoredMatrix_py.html","relative_filename":"transformer_lens/FactoredMatrix.py"}},"d_af97b5493da09a14_HookedEncoder_py":{"hash":"8b1980ed46804b90fefe423d7fdee191","index":{"nums":[0,1,162,0,10,104,33,35],"html_filename":"d_af97b5493da09a14_HookedEncoder_py.html","relative_filename":"transformer_lens/HookedEncoder.py"}},"d_af97b5493da09a14_HookedEncoderDecoder_py":{"hash":"ced41e685cd841a10cfabecef98df15c","index":{"nums":[0,1,170,0,19,112,41,45],"html_filename":"d_af97b5493da09a14_HookedEncoderDecoder_py.html","relative_filename":"transformer_lens/HookedEncoderDecoder.py"}},"d_af97b5493da09a14_HookedTransformer_py":{"hash":"3d8cd71693b9f1f5f8fee98901f42321","index":{"nums":[0,1,733,0,182,454,77,177],"html_filename":"d_af97b5493da09a14_HookedTransformer_py.html","relative_filename":"transformer_lens/HookedTransformer.py"}},"d_af97b5493da09a14_HookedTransformerConfig_py":{"hash":"899ad730b9143df46ba29b92514d5f16","index":{"nums":[0,1,130,0,8,40,7,7],"html_filename":"d_af97b5493da09a14_HookedTransformerConfig_py.html","relative_filename":"transformer_lens/HookedTransformerConfig.py"}},"d_af97b5493da09a14_SVDInterpreter_py":{"hash":"8b9a2792be15afabc033f624683fed44","index":{"nums":[0,1,55,0,1,16,1,1],"html_filename":"d_af97b5493da09a14_SVDInterpreter_py.html","relative_filename":"transformer_lens/SVDInterpreter.py"}},"d_af97b5493da09a14___init___py":{"hash":"b5c172d6f5d0c163dd8d3d8a188ef8fc","index":{"nums":[0,1,20,0,0,0,0,0],"html_filename":"d_af97b5493da09a14___init___py.html","relative_filename":"transformer_lens/__init__.py"}},"d_db46118ef83ad831___init___py":{"hash":"0f5ac43371106089cb8859730506eb31","index":{"nums":[0,1,20,0,0,0,0,0],"html_filename":"d_db46118ef83ad831___init___py.html","relative_filename":"transformer_lens/components/__init__.py"}},"d_db46118ef83ad831_abstract_attention_py":{"hash":"1e34e6bd63d5f4553e7a154f5619a95d","index":{"nums":[0,1,220,0,27,84,18,22],"html_filename":"d_db46118ef83ad831_abstract_attention_py.html","relative_filename":"transformer_lens/components/abstract_attention.py"}},"d_db46118ef83ad831_attention_py":{"hash":"771ce2a06e2204a8ea0b5c09549b0faa","index":{"nums":[0,1,20,0,4,4,2,2],"html_filename":"d_db46118ef83ad831_attention_py.html","relative_filename":"transformer_lens/components/attention.py"}},"d_db46118ef83ad831_bert_block_py":{"hash":"1d3faf4dfb51c82ef0e70e027c4442f7","index":{"nums":[0,1,45,0,4,2,1,1],"html_filename":"d_db46118ef83ad831_bert_block_py.html","relative_filename":"transformer_lens/components/bert_block.py"}},"d_db46118ef83ad831_bert_embed_py":{"hash":"fbb9b2fc553abf0e4d8a0afa34f58817","index":{"nums":[0,1,30,0,0,2,0,0],"html_filename":"d_db46118ef83ad831_bert_embed_py.html","relative_filename":"transformer_lens/components/bert_embed.py"}},"d_db46118ef83ad831_bert_mlm_head_py":{"hash":"0c5ab7d0820de68c654488b524f45c4d","index":{"nums":[0,1,20,0,0,0,0,0],"html_filename":"d_db46118ef83ad831_bert_mlm_head_py.html","relative_filename":"transformer_lens/components/bert_mlm_head.py"}},"d_db46118ef83ad831_embed_py":{"hash":"e80917c423b62b1fffeee5455ec2d952","index":{"nums":[0,1,17,0,0,4,0,0],"html_filename":"d_db46118ef83ad831_embed_py.html","relative_filename":"transformer_lens/components/embed.py"}},"d_db46118ef83ad831_grouped_query_attention_py":{"hash":"81ede5e95b90ea52c9793e25f7291b48","index":{"nums":[0,1,53,0,4,16,0,0],"html_filename":"d_db46118ef83ad831_grouped_query_attention_py.html","relative_filename":"transformer_lens/components/grouped_query_attention.py"}},"d_db46118ef83ad831_layer_norm_py":{"hash":"5d34676f9227c3c3bd83d13ca4a762ec","index":{"nums":[0,1,25,0,1,4,1,1],"html_filename":"d_db46118ef83ad831_layer_norm_py.html","relative_filename":"transformer_lens/components/layer_norm.py"}},"d_db46118ef83ad831_layer_norm_pre_py":{"hash":"a488e1ab3adf495070bf5e03582eee99","index":{"nums":[0,1,19,0,1,2,1,1],"html_filename":"d_db46118ef83ad831_layer_norm_pre_py.html","relative_filename":"transformer_lens/components/layer_norm_pre.py"}},"d_37285d613390727b_can_be_used_as_mlp_py":{"hash":"199b3e1e1d8fc5533d79c3788e257832","index":{"nums":[0,1,30,0,1,6,1,1],"html_filename":"d_37285d613390727b_can_be_used_as_mlp_py.html","relative_filename":"transformer_lens/components/mlps/can_be_used_as_mlp.py"}},"d_37285d613390727b_gated_mlp_py":{"hash":"1b9d23e893279e5255ac14d69e026485","index":{"nums":[0,1,31,0,3,4,2,2],"html_filename":"d_37285d613390727b_gated_mlp_py.html","relative_filename":"transformer_lens/components/mlps/gated_mlp.py"}},"d_37285d613390727b_gated_mlp_4bit_py":{"hash":"e85f0742f0a4142be44d967799763a15","index":{"nums":[0,1,32,0,18,4,1,3],"html_filename":"d_37285d613390727b_gated_mlp_4bit_py.html","relative_filename":"transformer_lens/components/mlps/gated_mlp_4bit.py"}},"d_37285d613390727b_mlp_py":{"hash":"a3901498f90188ad75b2b640144e7070","index":{"nums":[0,1,25,0,0,2,0,0],"html_filename":"d_37285d613390727b_mlp_py.html","relative_filename":"transformer_lens/components/mlps/mlp.py"}},"d_37285d613390727b_moe_py":{"hash":"c69707a0407a7cacf0929fa3233216f9","index":{"nums":[0,1,59,0,1,6,1,1],"html_filename":"d_37285d613390727b_moe_py.html","relative_filename":"transformer_lens/components/mlps/moe.py"}},"d_db46118ef83ad831_pos_embed_py":{"hash":"3688508c2257f2f12a9e431f92fdcd2a","index":{"nums":[0,1,23,0,0,2,0,0],"html_filename":"d_db46118ef83ad831_pos_embed_py.html","relative_filename":"transformer_lens/components/pos_embed.py"}},"d_db46118ef83ad831_rms_norm_py":{"hash":"19408e8189b8bb5d095ae6bb05045ef1","index":{"nums":[0,1,23,0,2,4,2,2],"html_filename":"d_db46118ef83ad831_rms_norm_py.html","relative_filename":"transformer_lens/components/rms_norm.py"}},"d_db46118ef83ad831_rms_norm_pre_py":{"hash":"04bea994792415493e41118c6e6c4da2","index":{"nums":[0,1,18,0,1,2,1,1],"html_filename":"d_db46118ef83ad831_rms_norm_pre_py.html","relative_filename":"transformer_lens/components/rms_norm_pre.py"}},"d_db46118ef83ad831_t5_attention_py":{"hash":"b02b9250eaf1c84e129e8763676c2d48","index":{"nums":[0,1,50,0,3,12,3,3],"html_filename":"d_db46118ef83ad831_t5_attention_py.html","relative_filename":"transformer_lens/components/t5_attention.py"}},"d_db46118ef83ad831_t5_block_py":{"hash":"8efa9a61311e44c6263d8dd61588fac4","index":{"nums":[0,1,64,0,6,12,3,3],"html_filename":"d_db46118ef83ad831_t5_block_py.html","relative_filename":"transformer_lens/components/t5_block.py"}},"d_db46118ef83ad831_token_typed_embed_py":{"hash":"0265390bcc2c91e19e04aee0cc722a2b","index":{"nums":[0,1,12,0,0,0,0,0],"html_filename":"d_db46118ef83ad831_token_typed_embed_py.html","relative_filename":"transformer_lens/components/token_typed_embed.py"}},"d_db46118ef83ad831_transformer_block_py":{"hash":"8fa97e0fe78b9c672714f56402884a44","index":{"nums":[0,1,101,0,15,52,10,18],"html_filename":"d_db46118ef83ad831_transformer_block_py.html","relative_filename":"transformer_lens/components/transformer_block.py"}},"d_db46118ef83ad831_unembed_py":{"hash":"a662f40d2a2543805f0b9f0d06f5b5c4","index":{"nums":[0,1,14,0,0,0,0,0],"html_filename":"d_db46118ef83ad831_unembed_py.html","relative_filename":"transformer_lens/components/unembed.py"}},"d_af97b5493da09a14_evals_py":{"hash":"7d496aae736b2525c13fe25d5363328b","index":{"nums":[0,1,147,0,50,46,2,14],"html_filename":"d_af97b5493da09a14_evals_py.html","relative_filename":"transformer_lens/evals.py"}},"d_65d4430f90bfb219_activation_function_factory_py":{"hash":"c4701f74bdab03c4b2cf3f7cd8afd980","index":{"nums":[0,1,12,0,2,6,2,2],"html_filename":"d_65d4430f90bfb219_activation_function_factory_py.html","relative_filename":"transformer_lens/factories/activation_function_factory.py"}},"d_65d4430f90bfb219_mlp_factory_py":{"hash":"4822436b1bacb210c29f36d16e51b216","index":{"nums":[0,1,14,0,1,6,1,1],"html_filename":"d_65d4430f90bfb219_mlp_factory_py.html","relative_filename":"transformer_lens/factories/mlp_factory.py"}},"d_af97b5493da09a14_head_detector_py":{"hash":"373e4b6b6945f7eec6de3b6987c4a9de","index":{"nums":[0,1,84,0,2,32,2,2],"html_filename":"d_af97b5493da09a14_head_detector_py.html","relative_filename":"transformer_lens/head_detector.py"}},"d_af97b5493da09a14_hook_points_py":{"hash":"cb760187bbf7a69669c316b56e7f9b12","index":{"nums":[0,1,234,0,41,124,19,49],"html_filename":"d_af97b5493da09a14_hook_points_py.html","relative_filename":"transformer_lens/hook_points.py"}},"d_af97b5493da09a14_loading_from_pretrained_py":{"hash":"d4b3e98ef63fb7fe1c8ecba8068b8715","index":{"nums":[0,1,308,0,107,204,47,89],"html_filename":"d_af97b5493da09a14_loading_from_pretrained_py.html","relative_filename":"transformer_lens/loading_from_pretrained.py"}},"d_af97b5493da09a14_past_key_value_caching_py":{"hash":"994a13780a7f92b6a2ab53a37e662c15","index":{"nums":[0,1,46,0,0,18,2,2],"html_filename":"d_af97b5493da09a14_past_key_value_caching_py.html","relative_filename":"transformer_lens/past_key_value_caching.py"}},"d_af97b5493da09a14_patching_py":{"hash":"d96c96e4ecf5fcfe45d2838d27a9d918","index":{"nums":[0,1,140,0,70,18,0,14],"html_filename":"d_af97b5493da09a14_patching_py.html","relative_filename":"transformer_lens/patching.py"}},"d_712808f24eb400fe___init___py":{"hash":"ca2411bb2d5283cba14da3ce60f5feec","index":{"nums":[0,1,0,0,0,0,0,0],"html_filename":"d_712808f24eb400fe___init___py.html","relative_filename":"transformer_lens/pretrained/__init__.py"}},"d_c1ea89878f9b2ac7___init___py":{"hash":"2994f6b388ff62b0d84269971b0766e3","index":{"nums":[0,1,20,0,0,0,0,0],"html_filename":"d_c1ea89878f9b2ac7___init___py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/__init__.py"}},"d_c1ea89878f9b2ac7_bert_py":{"hash":"1de2f9e5bcd80d6bcc6794f7073bbe66","index":{"nums":[0,1,31,0,0,2,0,0],"html_filename":"d_c1ea89878f9b2ac7_bert_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/bert.py"}},"d_c1ea89878f9b2ac7_bloom_py":{"hash":"28eee61cd9581bbb556c20abd44717d1","index":{"nums":[0,1,40,0,0,2,0,0],"html_filename":"d_c1ea89878f9b2ac7_bloom_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/bloom.py"}},"d_c1ea89878f9b2ac7_coder_py":{"hash":"008f8c07ead34001e59d834e9672b208","index":{"nums":[0,1,43,0,38,2,0,2],"html_filename":"d_c1ea89878f9b2ac7_coder_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/coder.py"}},"d_c1ea89878f9b2ac7_gemma_py":{"hash":"b3ee2563edaf1aec4f1db3777cf34c84","index":{"nums":[0,1,41,0,36,6,0,6],"html_filename":"d_c1ea89878f9b2ac7_gemma_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/gemma.py"}},"d_c1ea89878f9b2ac7_gpt2_py":{"hash":"b6864d10cf3b70f76be00e47695dbec2","index":{"nums":[0,1,39,0,0,2,0,0],"html_filename":"d_c1ea89878f9b2ac7_gpt2_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/gpt2.py"}},"d_c1ea89878f9b2ac7_gptj_py":{"hash":"6b6d95ba6133fe6935f33c2431cec376","index":{"nums":[0,1,36,0,31,2,0,2],"html_filename":"d_c1ea89878f9b2ac7_gptj_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/gptj.py"}},"d_c1ea89878f9b2ac7_llama_py":{"hash":"b283de32d58e72f23d1dde8b2a54f278","index":{"nums":[0,1,45,0,39,8,0,8],"html_filename":"d_c1ea89878f9b2ac7_llama_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/llama.py"}},"d_c1ea89878f9b2ac7_mingpt_py":{"hash":"8afdb1ccd308a76b7af9ea71404ed112","index":{"nums":[0,1,40,0,0,2,0,0],"html_filename":"d_c1ea89878f9b2ac7_mingpt_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/mingpt.py"}},"d_c1ea89878f9b2ac7_mistral_py":{"hash":"f08ee3cdaa7c2dfb152ef82a4917497f","index":{"nums":[0,1,36,0,31,2,0,2],"html_filename":"d_c1ea89878f9b2ac7_mistral_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/mistral.py"}},"d_c1ea89878f9b2ac7_mixtral_py":{"hash":"bb26007d26637b02b3869d4733d0d6e8","index":{"nums":[0,1,37,0,32,4,0,4],"html_filename":"d_c1ea89878f9b2ac7_mixtral_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/mixtral.py"}},"d_c1ea89878f9b2ac7_nanogpt_py":{"hash":"e2ff3818963256534c9ca6a4ff6566a5","index":{"nums":[0,1,52,0,47,10,0,10],"html_filename":"d_c1ea89878f9b2ac7_nanogpt_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/nanogpt.py"}},"d_c1ea89878f9b2ac7_neel_solu_old_py":{"hash":"d96be68dd2fba5dbcba015e7f8b10fc1","index":{"nums":[0,1,17,0,0,12,2,2],"html_filename":"d_c1ea89878f9b2ac7_neel_solu_old_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/neel_solu_old.py"}},"d_c1ea89878f9b2ac7_neo_py":{"hash":"67f2d6bb1adda03d8d3eacbe5e1a0cb4","index":{"nums":[0,1,37,0,0,2,0,0],"html_filename":"d_c1ea89878f9b2ac7_neo_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/neo.py"}},"d_c1ea89878f9b2ac7_neox_py":{"hash":"fde01ee4309dd58bf3a6793ca3421f9e","index":{"nums":[0,1,34,0,0,2,0,0],"html_filename":"d_c1ea89878f9b2ac7_neox_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/neox.py"}},"d_c1ea89878f9b2ac7_opt_py":{"hash":"5bd938981ffdf863cddf1e54a3526615","index":{"nums":[0,1,40,0,0,2,0,0],"html_filename":"d_c1ea89878f9b2ac7_opt_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/opt.py"}},"d_c1ea89878f9b2ac7_phi_py":{"hash":"487a556d6755479e53397cfa39646490","index":{"nums":[0,1,41,0,37,2,0,2],"html_filename":"d_c1ea89878f9b2ac7_phi_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/phi.py"}},"d_c1ea89878f9b2ac7_phi3_py":{"hash":"9d833a74a1082fc28aa204b928511d91","index":{"nums":[0,1,35,0,30,2,0,2],"html_filename":"d_c1ea89878f9b2ac7_phi3_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/phi3.py"}},"d_c1ea89878f9b2ac7_qwen_py":{"hash":"5abcfa29a1d0456fb304a4b3b7ee2063","index":{"nums":[0,1,38,0,33,2,0,2],"html_filename":"d_c1ea89878f9b2ac7_qwen_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/qwen.py"}},"d_c1ea89878f9b2ac7_qwen2_py":{"hash":"4f2b989be42046944eb76e787f12f8a4","index":{"nums":[0,1,41,0,36,2,0,2],"html_filename":"d_c1ea89878f9b2ac7_qwen2_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/qwen2.py"}},"d_c1ea89878f9b2ac7_t5_py":{"hash":"fb47f0cb69cfc6778072b400fe6e53ef","index":{"nums":[0,1,33,0,0,4,0,0],"html_filename":"d_c1ea89878f9b2ac7_t5_py.html","relative_filename":"transformer_lens/pretrained/weight_conversions/t5.py"}},"d_af97b5493da09a14_train_py":{"hash":"74c4dc0140e6c628f623287b84708fb2","index":{"nums":[0,1,72,0,42,34,1,33],"html_filename":"d_af97b5493da09a14_train_py.html","relative_filename":"transformer_lens/train.py"}},"d_b2114f845e0399b7___init___py":{"hash":"ca2411bb2d5283cba14da3ce60f5feec","index":{"nums":[0,1,0,0,0,0,0,0],"html_filename":"d_b2114f845e0399b7___init___py.html","relative_filename":"transformer_lens/utilities/__init__.py"}},"d_b2114f845e0399b7_activation_functions_py":{"hash":"0926a004863506fd686e3d1e0ae5eb84","index":{"nums":[0,1,6,0,0,2,0,0],"html_filename":"d_b2114f845e0399b7_activation_functions_py.html","relative_filename":"transformer_lens/utilities/activation_functions.py"}},"d_b2114f845e0399b7_addmm_py":{"hash":"b1cb8300653ca4c3dd6a4f5c06408a84","index":{"nums":[0,1,10,0,0,0,0,0],"html_filename":"d_b2114f845e0399b7_addmm_py.html","relative_filename":"transformer_lens/utilities/addmm.py"}},"d_b2114f845e0399b7_attention_py":{"hash":"7f944a3a3a9cf8f8df0018d09fea2fae","index":{"nums":[0,1,10,0,0,0,0,0],"html_filename":"d_b2114f845e0399b7_attention_py.html","relative_filename":"transformer_lens/utilities/attention.py"}},"d_b2114f845e0399b7_devices_py":{"hash":"1da1c1a01ae3a7697bbb04016881df11","index":{"nums":[0,1,31,0,7,18,4,10],"html_filename":"d_b2114f845e0399b7_devices_py.html","relative_filename":"transformer_lens/utilities/devices.py"}},"d_af97b5493da09a14_utils_py":{"hash":"6f14e8d0ec39c4d104978baee513b6e6","index":{"nums":[0,1,438,0,122,184,16,78],"html_filename":"d_af97b5493da09a14_utils_py.html","relative_filename":"transformer_lens/utils.py"}}}}
\ No newline at end of file
diff --git a/generated/code/transformer_lens.HookedTransformer.html b/generated/code/transformer_lens.HookedTransformer.html
index 34039dc47..6653c5e71 100644
--- a/generated/code/transformer_lens.HookedTransformer.html
+++ b/generated/code/transformer_lens.HookedTransformer.html
@@ -658,7 +658,7 @@
 
 <dl class="py method">
 <dt class="sig sig-object py" id="transformer_lens.HookedTransformer.HookedTransformer.from_pretrained">
-<em class="property"><span class="pre">classmethod</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">from_pretrained</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">model_name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">fold_ln</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">True</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">center_writing_weights</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">True</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">center_unembed</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">True</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">refactor_factored_attn_matrices</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">False</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">checkpoint_index</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Optional</span><span class="p"><span class="pre">[</span></span><span class="pre">int</span><span class="p"><span class="pre">]</span></span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">checkpoint_value</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Optional</span><span class="p"><span class="pre">[</span></span><span class="pre">int</span><span class="p"><span class="pre">]</span></span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">hf_model</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Optional</span><span class="p"><span class="pre">[</span></span><span class="pre">AutoModelForCausalLM</span><span class="p"><span class="pre">]</span></span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">device</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Optional</span><span class="p"><span class="pre">[</span></span><span class="pre">Union</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="pre">device</span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">]</span></span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">n_devices</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">int</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">1</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">tokenizer</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Optional</span><span class="p"><span class="pre">[</span></span><span class="pre">PreTrainedTokenizerBase</span><span class="p"><span class="pre">]</span></span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">move_to_device</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">True</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">fold_value_biases</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">True</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">default_prepend_bos</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">True</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">default_padding_side</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Literal</span><span class="p"><span class="pre">[</span></span><span class="s"><span class="pre">'left'</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="s"><span class="pre">'right'</span></span><span class="p"><span class="pre">]</span></span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">'right'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">dtype</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">'float32'</span></span></em>, <em class="sig-param"><span class="o"><span class="pre">**</span></span><span class="n"><span class="pre">from_pretrained_kwargs</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">&#x2192;</span> <span class="sig-return-typehint"><a class="reference internal" href="#transformer_lens.HookedTransformer.HookedTransformer" title="transformer_lens.HookedTransformer.HookedTransformer"><span class="pre">HookedTransformer</span></a></span></span><a class="headerlink" href="#transformer_lens.HookedTransformer.HookedTransformer.from_pretrained" title="Permalink to this definition">#</a></dt>
+<em class="property"><span class="pre">classmethod</span><span class="w"> </span></em><span class="sig-name descname"><span class="pre">from_pretrained</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">model_name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">fold_ln</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">True</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">center_writing_weights</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">True</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">center_unembed</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">True</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">refactor_factored_attn_matrices</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">False</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">checkpoint_index</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Optional</span><span class="p"><span class="pre">[</span></span><span class="pre">int</span><span class="p"><span class="pre">]</span></span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">checkpoint_value</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Optional</span><span class="p"><span class="pre">[</span></span><span class="pre">int</span><span class="p"><span class="pre">]</span></span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">hf_model</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Optional</span><span class="p"><span class="pre">[</span></span><span class="pre">AutoModelForCausalLM</span><span class="p"><span class="pre">]</span></span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">device</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Optional</span><span class="p"><span class="pre">[</span></span><span class="pre">Union</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="pre">device</span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">]</span></span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">n_devices</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">int</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">1</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">tokenizer</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Optional</span><span class="p"><span class="pre">[</span></span><span class="pre">PreTrainedTokenizerBase</span><span class="p"><span class="pre">]</span></span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">move_to_device</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">True</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">fold_value_biases</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">True</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">default_prepend_bos</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">True</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">default_padding_side</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Literal</span><span class="p"><span class="pre">[</span></span><span class="s"><span class="pre">'left'</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="s"><span class="pre">'right'</span></span><span class="p"><span class="pre">]</span></span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">'right'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">dtype</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">'float32'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">first_n_layers</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Optional</span><span class="p"><span class="pre">[</span></span><span class="pre">int</span><span class="p"><span class="pre">]</span></span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="o"><span class="pre">**</span></span><span class="n"><span class="pre">from_pretrained_kwargs</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">&#x2192;</span> <span class="sig-return-typehint"><a class="reference internal" href="#transformer_lens.HookedTransformer.HookedTransformer" title="transformer_lens.HookedTransformer.HookedTransformer"><span class="pre">HookedTransformer</span></a></span></span><a class="headerlink" href="#transformer_lens.HookedTransformer.HookedTransformer.from_pretrained" title="Permalink to this definition">#</a></dt>
 <dd><p>Load in a Pretrained Model.</p>
 <p>Load in pretrained model weights to the HookedTransformer format and optionally to do some
 processing to make the model easier to interpret. Currently supports loading from most
@@ -780,6 +780,7 @@
 the model.</p></li>
 <li><p><strong>default_padding_side</strong> – Which side to pad on when tokenizing. Defaults to
 “right”.</p></li>
+<li><p><strong>first_n_layers</strong> – If specified, only load the first n layers of the model.</p></li>
 </ul>
 </dd>
 </dl>
diff --git a/generated/code/transformer_lens.HookedTransformerConfig.html b/generated/code/transformer_lens.HookedTransformerConfig.html
index 8c5709294..0c6ef6723 100644
--- a/generated/code/transformer_lens.HookedTransformerConfig.html
+++ b/generated/code/transformer_lens.HookedTransformerConfig.html
@@ -360,7 +360,7 @@
 attention</p></li>
 <li><p><strong>attn_types</strong> (List[str], <em>optional</em>) – the types of attention to use for
 local attention</p></li>
-<li><p><strong>weight_init_mode</strong> (<em>str</em>) – the initialization mode to use for the
+<li><p><strong>init_mode</strong> (<em>str</em>) – the initialization mode to use for the
 weights. Only relevant for custom models, ignored for pre-trained.
 We now support ‘gpt2’, ‘xavier_uniform’, ‘xavier_normal’, ‘kaiming_uniform’,
 ‘kaiming_normal’. MuP support to come. Defaults to ‘gpt2’.</p></li>
@@ -382,7 +382,7 @@
 Used to set sources of randomness (Python, PyTorch and NumPy) and to initialize weights.
 Defaults to None. We recommend setting a seed, so your experiments are reproducible.</p></li>
 <li><p><strong>initializer_range</strong> (<em>float</em>) – The standard deviation of the normal used to
-initialise the weights, initialized to 0.8 / sqrt(d_model). If weight_init_mode is
+initialise the weights, initialized to 0.8 / sqrt(d_model). If init_mode is
 ‘xavier_uniform’ or ‘xavier_normal’, this value is instead treated as the <cite>gain</cite> parameter for the weight
 initialisation (a constant factor to scale the weights by). Defaults to -1.0, which means not set.</p></li>
 <li><p><strong>init_weights</strong> (<em>bool</em>) – Whether to initialize the weights. Defaults to
diff --git a/generated/code/transformer_lens.loading_from_pretrained.html b/generated/code/transformer_lens.loading_from_pretrained.html
index f3b45208d..b2202e4c4 100644
--- a/generated/code/transformer_lens.loading_from_pretrained.html
+++ b/generated/code/transformer_lens.loading_from_pretrained.html
@@ -389,7 +389,7 @@
 
 <dl class="py function">
 <dt class="sig sig-object py" id="transformer_lens.loading_from_pretrained.get_pretrained_model_config">
-<span class="sig-prename descclassname"><span class="pre">transformer_lens.loading_from_pretrained.</span></span><span class="sig-name descname"><span class="pre">get_pretrained_model_config</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">model_name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">hf_cfg</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Optional</span><span class="p"><span class="pre">[</span></span><span class="pre">dict</span><span class="p"><span class="pre">]</span></span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">checkpoint_index</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Optional</span><span class="p"><span class="pre">[</span></span><span class="pre">int</span><span class="p"><span class="pre">]</span></span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">checkpoint_value</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Optional</span><span class="p"><span class="pre">[</span></span><span class="pre">int</span><span class="p"><span class="pre">]</span></span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">fold_ln</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">False</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">device</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Optional</span><span class="p"><span class="pre">[</span></span><span class="pre">Union</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="pre">device</span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">]</span></span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">n_devices</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">int</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">1</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">default_prepend_bos</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">True</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">dtype</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">dtype</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">torch.float32</span></span></em>, <em class="sig-param"><span class="o"><span class="pre">**</span></span><span class="n"><span class="pre">kwargs</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#transformer_lens.loading_from_pretrained.get_pretrained_model_config" title="Permalink to this definition">#</a></dt>
+<span class="sig-prename descclassname"><span class="pre">transformer_lens.loading_from_pretrained.</span></span><span class="sig-name descname"><span class="pre">get_pretrained_model_config</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">model_name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">hf_cfg</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Optional</span><span class="p"><span class="pre">[</span></span><span class="pre">dict</span><span class="p"><span class="pre">]</span></span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">checkpoint_index</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Optional</span><span class="p"><span class="pre">[</span></span><span class="pre">int</span><span class="p"><span class="pre">]</span></span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">checkpoint_value</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Optional</span><span class="p"><span class="pre">[</span></span><span class="pre">int</span><span class="p"><span class="pre">]</span></span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">fold_ln</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">False</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">device</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Optional</span><span class="p"><span class="pre">[</span></span><span class="pre">Union</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="pre">device</span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">]</span></span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">n_devices</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">int</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">1</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">default_prepend_bos</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">True</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">dtype</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">dtype</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">torch.float32</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">first_n_layers</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Optional</span><span class="p"><span class="pre">[</span></span><span class="pre">int</span><span class="p"><span class="pre">]</span></span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="o"><span class="pre">**</span></span><span class="n"><span class="pre">kwargs</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#transformer_lens.loading_from_pretrained.get_pretrained_model_config" title="Permalink to this definition">#</a></dt>
 <dd><p>Returns the pretrained model config as an HookedTransformerConfig object.</p>
 <p>There are two types of pretrained models: HuggingFace models (where
 AutoModel and AutoConfig work), and models trained by me (NeelNanda) which
diff --git a/generated/demos/Exploratory_Analysis_Demo.html b/generated/demos/Exploratory_Analysis_Demo.html
index 4318981cb..9f5dd2a70 100644
--- a/generated/demos/Exploratory_Analysis_Demo.html
+++ b/generated/demos/Exploratory_Analysis_Demo.html
@@ -476,37 +476,37 @@ <h3>Indirect Object Identification<a class="headerlink" href="#Indirect-Object-I
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area docutils container">
-<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "a42e2bbeb4644be780fd859fae156453"}</script></div>
+<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "515362cd489440da86ee9da4b18234d4"}</script></div>
 </div>
 <div class="nboutput docutils container">
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area docutils container">
-<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "f98e0318bcde48c7bc9ff7fa4c098c64"}</script></div>
+<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "6325ae084ee34b078afc0e06f01c6afa"}</script></div>
 </div>
 <div class="nboutput docutils container">
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area docutils container">
-<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "38bfcf873a8a4d7fbf3002998c2b9e9e"}</script></div>
+<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "12c7e81fccbe41dda672b6b6722e8f98"}</script></div>
 </div>
 <div class="nboutput docutils container">
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area docutils container">
-<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "1184e705dd634ff29325f8222cc7406e"}</script></div>
+<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "f4d8cb4603f64164bba7cbecd423516d"}</script></div>
 </div>
 <div class="nboutput docutils container">
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area docutils container">
-<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "b9a1fa123e9f4bd89dad50aac41fcff5"}</script></div>
+<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "27abca058a2d442c9fa1488c14f5fe1c"}</script></div>
 </div>
 <div class="nboutput docutils container">
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area docutils container">
-<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "b35dabf61035460aa1dd0200cf89fa93"}</script></div>
+<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "f10863d1bbd14b91819db0b9074aa85a"}</script></div>
 </div>
 <div class="nboutput nblast docutils container">
 <div class="prompt empty docutils container">
@@ -899,9 +899,9 @@ <h3>Logit Lens<a class="headerlink" href="#Logit-Lens" title="Permalink to this
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area rendered_html docutils container">
-<div>                            <div id="54c9da27-b5e5-42aa-baed-1fe18fb97e41" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("54c9da27-b5e5-42aa-baed-1fe18fb97e41")) {                    Plotly.newPlot(                        "54c9da27-b5e5-42aa-baed-1fe18fb97e41",                        [{"hovertemplate":"\u003cb\u003e%{hovertext}\u003c\u002fb\u003e\u003cbr\u003e\u003cbr\u003ex=%{x}\u003cbr\u003ey=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","hovertext":["0_pre","0_mid","1_pre","1_mid","2_pre","2_mid","3_pre","3_mid","4_pre","4_mid","5_pre","5_mid","6_pre","6_mid","7_pre","7_mid","8_pre","8_mid","9_pre","9_mid","10_pre","10_mid","11_pre","11_mid","final_post"],"legendgroup":"","line":{"color":"#636efa","dash":"solid"},"marker":{"symbol":"circle"},"mode":"lines","name":"","orientation":"v","showlegend":false,"x":[0.0,0.5,1.0,1.5,2.0,2.5,3.0,3.5,4.0,4.5,5.0,5.5,6.0,6.5,7.0,7.5,8.0,8.5,9.0,9.5,10.0,10.5,11.0,11.5,12.0],"xaxis":"x","y":[1.2936594430357218e-05,-0.0066434419713914394,-0.007525080814957619,-0.009075661189854145,-0.008736724965274334,-0.008685516193509102,-0.006480449344962835,-0.007939981296658516,-0.009662019088864326,-0.01509616058319807,-0.014190619811415672,-0.019929546862840652,-0.009124485775828362,-0.02729814499616623,-0.02985497936606407,0.24972598254680634,0.25055843591690063,0.4500514566898346,0.4599689245223999,5.025448322296143,5.142899036407471,4.73056697845459,4.887067794799805,3.4453935623168945,3.5518839359283447],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"x"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"y"}},"legend":{"tracegroupgap":0},"title":{"text":"Logit Difference From Accumulate Residual Stream"}},                        {"responsive": true}                    ).then(function(){
+<div>                            <div id="3d63aabd-d814-4fc0-b7e4-841c761b56cd" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("3d63aabd-d814-4fc0-b7e4-841c761b56cd")) {                    Plotly.newPlot(                        "3d63aabd-d814-4fc0-b7e4-841c761b56cd",                        [{"hovertemplate":"\u003cb\u003e%{hovertext}\u003c\u002fb\u003e\u003cbr\u003e\u003cbr\u003ex=%{x}\u003cbr\u003ey=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","hovertext":["0_pre","0_mid","1_pre","1_mid","2_pre","2_mid","3_pre","3_mid","4_pre","4_mid","5_pre","5_mid","6_pre","6_mid","7_pre","7_mid","8_pre","8_mid","9_pre","9_mid","10_pre","10_mid","11_pre","11_mid","final_post"],"legendgroup":"","line":{"color":"#636efa","dash":"solid"},"marker":{"symbol":"circle"},"mode":"lines","name":"","orientation":"v","showlegend":false,"x":[0.0,0.5,1.0,1.5,2.0,2.5,3.0,3.5,4.0,4.5,5.0,5.5,6.0,6.5,7.0,7.5,8.0,8.5,9.0,9.5,10.0,10.5,11.0,11.5,12.0],"xaxis":"x","y":[1.2936594430357218e-05,-0.0066434419713914394,-0.007525080814957619,-0.009075661189854145,-0.008736724965274334,-0.008685516193509102,-0.006480449344962835,-0.007939981296658516,-0.009662019088864326,-0.01509616058319807,-0.014190619811415672,-0.019929546862840652,-0.009124485775828362,-0.02729814499616623,-0.02985497936606407,0.24972598254680634,0.25055843591690063,0.4500514566898346,0.4599689245223999,5.025448322296143,5.142899036407471,4.73056697845459,4.887067794799805,3.4453935623168945,3.5518839359283447],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"x"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"y"}},"legend":{"tracegroupgap":0},"title":{"text":"Logit Difference From Accumulate Residual Stream"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('54c9da27-b5e5-42aa-baed-1fe18fb97e41');
+var gd = document.getElementById('3d63aabd-d814-4fc0-b7e4-841c761b56cd');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -947,9 +947,9 @@ <h3>Layer Attribution<a class="headerlink" href="#Layer-Attribution" title="Perm
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area rendered_html docutils container">
-<div>                            <div id="13c88f1d-dc19-46a7-ac53-d34066484b06" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("13c88f1d-dc19-46a7-ac53-d34066484b06")) {                    Plotly.newPlot(                        "13c88f1d-dc19-46a7-ac53-d34066484b06",                        [{"hovertemplate":"\u003cb\u003e%{hovertext}\u003c\u002fb\u003e\u003cbr\u003e\u003cbr\u003ex=%{x}\u003cbr\u003ey=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","hovertext":["embed","pos_embed","0_attn_out","0_mlp_out","1_attn_out","1_mlp_out","2_attn_out","2_mlp_out","3_attn_out","3_mlp_out","4_attn_out","4_mlp_out","5_attn_out","5_mlp_out","6_attn_out","6_mlp_out","7_attn_out","7_mlp_out","8_attn_out","8_mlp_out","9_attn_out","9_mlp_out","10_attn_out","10_mlp_out","11_attn_out","11_mlp_out"],"legendgroup":"","line":{"color":"#636efa","dash":"solid"},"marker":{"symbol":"circle"},"mode":"lines","name":"","orientation":"v","showlegend":false,"x":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25],"xaxis":"x","y":[-0.00028366505284793675,0.00029660051222890615,-0.006656361743807793,-0.0008817464113235474,-0.0015505198389291763,0.0003389097983017564,5.121063441038132e-05,0.002205097349360585,-0.001459577470086515,-0.0017220661975443363,-0.0054340846836566925,0.0009054935071617365,-0.005738964769989252,0.010805057361721992,-0.018173594027757645,-0.002556750550866127,0.27958089113235474,0.0008324328809976578,0.19949296116828918,0.009917395189404488,4.565478801727295,0.11744903028011322,-0.41232579946517944,0.1564953327178955,-1.4416704177856445,0.10648898035287857],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"x"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"y"}},"legend":{"tracegroupgap":0},"title":{"text":"Logit Difference From Each Layer"}},                        {"responsive": true}                    ).then(function(){
+<div>                            <div id="8309d7f8-599e-48e5-859f-a6ec1ae1a080" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("8309d7f8-599e-48e5-859f-a6ec1ae1a080")) {                    Plotly.newPlot(                        "8309d7f8-599e-48e5-859f-a6ec1ae1a080",                        [{"hovertemplate":"\u003cb\u003e%{hovertext}\u003c\u002fb\u003e\u003cbr\u003e\u003cbr\u003ex=%{x}\u003cbr\u003ey=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","hovertext":["embed","pos_embed","0_attn_out","0_mlp_out","1_attn_out","1_mlp_out","2_attn_out","2_mlp_out","3_attn_out","3_mlp_out","4_attn_out","4_mlp_out","5_attn_out","5_mlp_out","6_attn_out","6_mlp_out","7_attn_out","7_mlp_out","8_attn_out","8_mlp_out","9_attn_out","9_mlp_out","10_attn_out","10_mlp_out","11_attn_out","11_mlp_out"],"legendgroup":"","line":{"color":"#636efa","dash":"solid"},"marker":{"symbol":"circle"},"mode":"lines","name":"","orientation":"v","showlegend":false,"x":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25],"xaxis":"x","y":[-0.00028366505284793675,0.00029660051222890615,-0.006656361743807793,-0.0008817464113235474,-0.0015505198389291763,0.0003389097983017564,5.121063441038132e-05,0.002205097349360585,-0.001459577470086515,-0.0017220661975443363,-0.0054340846836566925,0.0009054935071617365,-0.005738964769989252,0.010805057361721992,-0.018173594027757645,-0.002556750550866127,0.27958089113235474,0.0008324328809976578,0.19949296116828918,0.009917395189404488,4.565478801727295,0.11744903028011322,-0.41232579946517944,0.1564953327178955,-1.4416704177856445,0.10648898035287857],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"x"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"y"}},"legend":{"tracegroupgap":0},"title":{"text":"Logit Difference From Each Layer"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('13c88f1d-dc19-46a7-ac53-d34066484b06');
+var gd = document.getElementById('8309d7f8-599e-48e5-859f-a6ec1ae1a080');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -1017,9 +1017,9 @@ <h2>Head Attribution<a class="headerlink" href="#Head-Attribution" title="Permal
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area rendered_html docutils container">
-<div>                            <div id="c34feb68-85f6-4bc2-84d3-a7697a1138a3" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("c34feb68-85f6-4bc2-84d3-a7697a1138a3")) {                    Plotly.newPlot(                        "c34feb68-85f6-4bc2-84d3-a7697a1138a3",                        [{"coloraxis":"coloraxis","name":"0","z":[[-0.0020563285797834396,-0.0005101955030113459,0.0004685768508352339,0.00012511832755990326,-0.0006028721109032631,-0.00024295502225868404,-0.002318894723430276,-0.002758359769359231,0.0005645868368446827,0.000969740329310298,-0.0002504501899238676,4.740082658827305e-06],[-0.0010071131400763988,0.0003947088844142854,-0.0015487205237150192,0.0014034901978448033,-0.0012652688892558217,-0.0011358250631019473,-0.0028159404173493385,-0.0029645359609276056,0.0029190238565206528,0.0025743518490344286,0.00036237656604498625,0.0017548884497955441],[0.0005569332861341536,-0.00112663593608886,-0.0017354178708046675,-0.0014514722861349583,-0.00028740704874508083,0.0017210595542564988,0.00266590085811913,0.0031146793626248837,0.0005667305667884648,-0.0036664949730038643,-0.0018847067840397358,7.027178071439266e-06],[-0.000726439815480262,0.00011369686399120837,0.001430142787285149,0.000749052269384265,0.0020184761378914118,0.0007436758605763316,-0.0004617759259417653,-0.003905785735696554,0.001140733016654849,-4.027335671707988e-05,-0.0013293256051838398,-0.0017636881675571203],[-0.002828173339366913,0.0003364472358953208,-0.0014249038649722934,-0.0037773081567138433,0.0015998876187950373,0.00029889732832089067,-0.0008046309230849147,0.0020388164557516575,-0.0015593776479363441,-0.0006437147385440767,0.001116806990467012,-0.00035003889934159815],[0.0011338151525706053,0.0011259106686338782,-0.0025163597892969847,-0.0014790240675210953,0.00038791983388364315,-6.407807813957334e-05,-0.0005096746608614922,-0.0008841876406222582,0.0006399309495463967,-0.001009696745313704,-0.0067590330727398396,0.0033667273819446564],[-0.015147387981414795,-0.002135086804628372,0.002593189012259245,-0.00042674108408391476,-0.005559002980589867,0.0026659294962882996,0.006410874892026186,-0.0038270100485533476,-0.00038422830402851105,-0.0016430210089311004,-0.0013344308827072382,-9.184109512716532e-05],[-9.488123760093004e-05,-0.005788922309875488,-0.0006383719155564904,0.134933739900589,-0.0017687628278508782,-0.018917974084615707,0.0038733629044145346,-0.002145076170563698,-0.010327237658202648,0.18325874209403992,-0.0007747883792035282,-0.0010452116839587688],[-0.003833947703242302,-0.000804627372417599,-0.012673338875174522,0.008045812137424946,0.0036040153354406357,-0.009398169815540314,-0.08272106945514679,0.0035550352185964584,-0.018404126167297363,0.0017586719477549195,0.2896132469177246,0.022854045033454895],[0.08595201373100281,-0.0006932567339390516,0.06816966831684113,0.01311141811311245,-0.021098004654049873,0.05112440511584282,1.384489893913269,0.04583733528852463,-0.038303446024656296,2.9854445457458496,0.001966139767318964,-0.008030213415622711],[0.560872495174408,0.17083144187927246,-0.033618733286857605,0.05821547657251358,-0.0024530075024813414,0.0018771879840642214,0.2882729768753052,-1.898641586303711,-0.001528693363070488,-0.03513003885746002,0.48021769523620605,-0.0009116916917264462],[0.0160758625715971,-0.03986112400889397,-0.3879111707210541,0.011123226955533028,-0.005477802362293005,-0.0025129495188593864,-0.08056114614009857,0.007518642581999302,0.043011054396629333,-0.0400824099779129,-0.9702335596084595,0.011862391605973244]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Logit Difference From Each Head"}},                        {"responsive": true}                    ).then(function(){
+<div>                            <div id="3827203b-16c0-45c2-aab4-743cf2cb601f" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("3827203b-16c0-45c2-aab4-743cf2cb601f")) {                    Plotly.newPlot(                        "3827203b-16c0-45c2-aab4-743cf2cb601f",                        [{"coloraxis":"coloraxis","name":"0","z":[[-0.0020563285797834396,-0.0005101955030113459,0.0004685768508352339,0.00012511832755990326,-0.0006028721109032631,-0.00024295502225868404,-0.002318894723430276,-0.002758359769359231,0.0005645868368446827,0.000969740329310298,-0.0002504501899238676,4.740082658827305e-06],[-0.0010071131400763988,0.0003947088844142854,-0.0015487205237150192,0.0014034901978448033,-0.0012652688892558217,-0.0011358250631019473,-0.0028159404173493385,-0.0029645359609276056,0.0029190238565206528,0.0025743518490344286,0.00036237656604498625,0.0017548884497955441],[0.0005569332861341536,-0.00112663593608886,-0.0017354178708046675,-0.0014514722861349583,-0.00028740704874508083,0.0017210595542564988,0.00266590085811913,0.0031146793626248837,0.0005667305667884648,-0.0036664949730038643,-0.0018847067840397358,7.027178071439266e-06],[-0.000726439815480262,0.00011369686399120837,0.001430142787285149,0.000749052269384265,0.0020184761378914118,0.0007436758605763316,-0.0004617759259417653,-0.003905785735696554,0.001140733016654849,-4.027335671707988e-05,-0.0013293256051838398,-0.0017636881675571203],[-0.002828173339366913,0.0003364472358953208,-0.0014249038649722934,-0.0037773081567138433,0.0015998876187950373,0.00029889732832089067,-0.0008046309230849147,0.0020388164557516575,-0.0015593776479363441,-0.0006437147385440767,0.001116806990467012,-0.00035003889934159815],[0.0011338151525706053,0.0011259106686338782,-0.0025163597892969847,-0.0014790240675210953,0.00038791983388364315,-6.407807813957334e-05,-0.0005096746608614922,-0.0008841876406222582,0.0006399309495463967,-0.001009696745313704,-0.0067590330727398396,0.0033667273819446564],[-0.015147387981414795,-0.002135086804628372,0.002593189012259245,-0.00042674108408391476,-0.005559002980589867,0.0026659294962882996,0.006410874892026186,-0.0038270100485533476,-0.00038422830402851105,-0.0016430210089311004,-0.0013344308827072382,-9.184109512716532e-05],[-9.488123760093004e-05,-0.005788922309875488,-0.0006383719155564904,0.134933739900589,-0.0017687628278508782,-0.018917974084615707,0.0038733629044145346,-0.002145076170563698,-0.010327237658202648,0.18325874209403992,-0.0007747883792035282,-0.0010452116839587688],[-0.003833947703242302,-0.000804627372417599,-0.012673338875174522,0.008045812137424946,0.0036040153354406357,-0.009398169815540314,-0.08272106945514679,0.0035550352185964584,-0.018404126167297363,0.0017586719477549195,0.2896132469177246,0.022854045033454895],[0.08595201373100281,-0.0006932567339390516,0.06816966831684113,0.01311141811311245,-0.021098004654049873,0.05112440511584282,1.384489893913269,0.04583733528852463,-0.038303446024656296,2.9854445457458496,0.001966139767318964,-0.008030213415622711],[0.560872495174408,0.17083144187927246,-0.033618733286857605,0.05821547657251358,-0.0024530075024813414,0.0018771879840642214,0.2882729768753052,-1.898641586303711,-0.001528693363070488,-0.03513003885746002,0.48021769523620605,-0.0009116916917264462],[0.0160758625715971,-0.03986112400889397,-0.3879111707210541,0.011123226955533028,-0.005477802362293005,-0.0025129495188593864,-0.08056114614009857,0.007518642581999302,0.043011054396629333,-0.0400824099779129,-0.9702335596084595,0.011862391605973244]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Logit Difference From Each Head"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('c34feb68-85f6-4bc2-84d3-a7697a1138a3');
+var gd = document.getElementById('3827203b-16c0-45c2-aab4-743cf2cb601f');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -1141,19 +1141,19 @@ <h2>Attention Analysis<a class="headerlink" href="#Attention-Analysis" title="Pe
 </pre></div>
 </div>
 <div class="output_area rendered_html docutils container">
-<div style='max-width: 700px;'><h2>Top 3 Positive Logit Attribution Heads</h2><br/><div id="circuits-vis-6d53690c-9574" style="margin: 15px 0;"/>
+<div style='max-width: 700px;'><h2>Top 3 Positive Logit Attribution Heads</h2><br/><div id="circuits-vis-640204d8-20cc" style="margin: 15px 0;"/>
     <script crossorigin type="module">
     import { render, AttentionHeads } from "https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js";
     render(
-      "circuits-vis-6d53690c-9574",
+      "circuits-vis-640204d8-20cc",
       AttentionHeads,
       {"attention": [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9979304075241089, 0.002069620881229639, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9971864819526672, 0.0010516609763726592, 0.0017618348356336355, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9594857096672058, 0.001310725579969585, 0.03694308176636696, 0.002260456094518304, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9891613125801086, 0.001052629784680903, 0.0048541901633143425, 0.001104357186704874, 0.003827564185485244, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9963598847389221, 0.0007981774397194386, 0.0007764195324853063, 0.00019262291607446969, 0.0002416159404674545, 0.0016313291853293777, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.981286346912384, 0.001863775309175253, 0.006344134453684092, 0.0002960922720376402, 0.004604941233992577, 0.0013966941041871905, 0.00420788861811161, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9950011372566223, 0.0013230532640591264, 0.0007920759962871671, 0.00024644460063427687, 0.0003405151073820889, 0.00016826140927150846, 0.0002837753272615373, 0.0018447580514475703, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9920291900634766, 0.0013539871433749795, 0.0007168339216150343, 9.214139572577551e-05, 0.00013419234892353415, 0.00019706363673321903, 0.00035278062568977475, 0.0002426079590804875, 0.004881155211478472, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5483196377754211, 0.0075838202610611916, 0.28265973925590515, 0.0029685653280466795, 0.14815661311149597, 0.0008172960951924324, 0.0006479909643530846, 0.0017078499076887965, 0.004129356704652309, 0.003009046893566847, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8642559051513672, 0.0036493062507361174, 0.004925842396914959, 0.06957440823316574, 0.038683053106069565, 0.009609837085008621, 0.0008548588375560939, 0.0005643004551529884, 0.003707138355821371, 0.0013952680164948106, 0.002780066104605794, 0.0, 0.0, 0.0, 0.0], [0.30252957344055176, 0.006584456656128168, 0.1401754468679428, 0.030065739527344704, 0.4806697964668274, 0.0005923578282818198, 0.0005309724947437644, 0.0024134721606969833, 0.008505754172801971, 0.0013766245683655143, 0.023764921352267265, 0.0027908117044717073, 0.0, 0.0, 0.0], [0.7773900032043457, 0.003722917754203081, 0.022378109395503998, 0.015139483846724033, 0.017408354207873344, 0.002416662173345685, 0.000709561922121793, 0.0007399603491649032, 0.13850137591362, 0.0023606533650308847, 0.005776867736130953, 0.0028014343697577715, 0.010654616169631481, 0.0, 0.0], [0.9735962152481079, 0.0012812522472813725, 0.002617918187752366, 9.891873924061656e-05, 0.0005098494002595544, 0.00012000725837424397, 0.00045229491661302745, 0.00010779645526781678, 0.002954537281766534, 0.001644388772547245, 0.0012141242623329163, 0.00018637391622178257, 0.0008510759216733277, 0.01436527632176876, 0.0], [0.0990300253033638, 0.000971083587501198, 0.06413238495588303, 0.005246965680271387, 0.8110400438308716, 8.250321843661368e-05, 6.596777529921383e-05, 0.0011991349747404456, 0.0008852760074660182, 0.00024891181965358555, 0.010336406528949738, 7.952339365147054e-05, 0.0049433172680437565, 0.00028918671887367964, 0.0014492359478026628]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9985041618347168, 0.0014959010295569897, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9966639876365662, 0.00046957843005657196, 0.002866449998691678, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9208283424377441, 0.0007845019572414458, 0.07676984369754791, 0.001617342815734446, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9835838675498962, 0.0006018307176418602, 0.0030122774187475443, 0.006188061088323593, 0.0066139851696789265, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9950478076934814, 0.0007798039587214589, 0.0007059435010887682, 0.0003824840241577476, 0.0007798775332048535, 0.0023041421081870794, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9838821887969971, 0.0015907409833744168, 0.0063361418433487415, 0.0004584550915751606, 0.004829846788197756, 0.0009966548532247543, 0.0019060741178691387, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.995481014251709, 0.0010436347220093012, 0.0005037460359744728, 0.0005756175960414112, 0.00046194688184186816, 0.0004267194017302245, 0.0006751787732355297, 0.0008320942288264632, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9798213243484497, 0.008518284186720848, 0.0010190301109105349, 0.00029932931647635996, 0.0003097986045759171, 0.0003330526524223387, 0.001541984616778791, 0.0008126517641358078, 0.007344536483287811, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6489241123199463, 0.008722440339624882, 0.10265976935625076, 0.0038905434776097536, 0.22694909572601318, 0.0011350901331752539, 0.00045881845289841294, 0.0004216691595502198, 0.0053738984279334545, 0.0014645822811871767, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9156036972999573, 0.0013435552828013897, 0.0011673923581838608, 0.04532041400671005, 0.020157871767878532, 0.007576430682092905, 0.0011305580846965313, 0.00024880640557967126, 0.004846310243010521, 0.0007249554619193077, 0.0018800381803885102, 0.0, 0.0, 0.0, 0.0], [0.4962114989757538, 0.0019782893359661102, 0.05844113230705261, 0.00783691368997097, 0.40375715494155884, 0.0005728096584789455, 0.0002963987353723496, 0.0009752871701493859, 0.0046507869847118855, 0.00032523463596589863, 0.02318551018834114, 0.0017689995002001524, 0.0, 0.0, 0.0], [0.8397698402404785, 0.0018636900931596756, 0.005577529780566692, 0.020380759611725807, 0.0292733795940876, 0.004448764491826296, 0.0011503315763548017, 0.00024191653938032687, 0.08715140074491501, 0.0013588638976216316, 0.0026416215114295483, 0.0019266613526269794, 0.004215260501950979, 0.0, 0.0], [0.9744170308113098, 0.0024021149147301912, 0.00455052312463522, 0.00020547708845697343, 0.002662503393366933, 0.00021827162709087133, 0.0008242498734034598, 0.0004054978198837489, 0.002643037121742964, 0.0015235628234222531, 0.0031088448595255613, 7.676492532482371e-05, 0.0018502527382224798, 0.0051117767579853535, 0.0], [0.18470405042171478, 0.00038850627606734633, 0.06518032401800156, 0.0014534658985212445, 0.7318763732910156, 0.00010888298129430041, 2.748575025179889e-05, 0.00011231198004679754, 0.0013672056375071406, 4.0162569348467514e-05, 0.012543701566755772, 5.95860110479407e-05, 0.0007756491540931165, 0.0010575354099273682, 0.00030486442847177386]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9776753783226013, 0.022324683144688606, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9430753588676453, 0.018823042511940002, 0.038101643323898315, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8963208794593811, 0.006171742454171181, 0.07676312327384949, 0.02074429951608181, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8522229790687561, 0.017844438552856445, 0.05478672310709953, 0.012976272031664848, 0.062169551849365234, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9669023752212524, 0.004049547016620636, 0.002062328392639756, 0.007766399998217821, 0.007081815041601658, 0.012137639336287975, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7676379680633545, 0.018502332270145416, 0.045013245195150375, 0.012471744790673256, 0.10591386258602142, 0.018323613330721855, 0.03213733434677124, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.891828179359436, 0.010069617070257664, 0.013090009801089764, 0.010685192421078682, 0.031169477850198746, 0.006502270698547363, 0.02103857509791851, 0.015616719610989094, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9457208514213562, 0.0045005157589912415, 0.013905955478549004, 0.0013248942559584975, 0.01557664293795824, 0.001115700462833047, 0.0007837467710487545, 0.0007244577864184976, 0.016347244381904602, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5131928324699402, 0.006032112520188093, 0.11406763643026352, 0.009519217535853386, 0.30613386631011963, 0.0034422571770846844, 0.002601674757897854, 0.002665397012606263, 0.03412351757287979, 0.008221502415835857, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8395131230354309, 0.008326224982738495, 0.012920893728733063, 0.01282776053994894, 0.029934609308838844, 0.013884111307561398, 0.0034839101135730743, 0.0018691613804548979, 0.039465710520744324, 0.011843967251479626, 0.025930611416697502, 0.0, 0.0, 0.0, 0.0], [0.3585011661052704, 0.004236206877976656, 0.10680253803730011, 0.015598620288074017, 0.41607996821403503, 0.002491925610229373, 0.003585703205317259, 0.0038999069947749376, 0.015854908153414726, 0.005938141606748104, 0.062027692794799805, 0.004983273800462484, 0.0, 0.0, 0.0], [0.612542450428009, 0.004679449368268251, 0.029155218973755836, 0.028240399435162544, 0.08258962631225586, 0.007909866981208324, 0.011129351332783699, 0.0034775957465171814, 0.17027747631072998, 0.010538897477090359, 0.015773506835103035, 0.013295088894665241, 0.01039104349911213, 0.0, 0.0], [0.875810444355011, 0.007089323364198208, 0.01208286453038454, 0.0025996132753789425, 0.021269435063004494, 0.00165357009973377, 0.0019036760786548257, 0.0016310046194121242, 0.02823040634393692, 0.008473552763462067, 0.011080354452133179, 0.002760963048785925, 0.0025940914638340473, 0.02282082475721836, 0.0], [0.5415229201316833, 0.003397508757188916, 0.03888256847858429, 0.007577123120427132, 0.3613935708999634, 0.001381068374030292, 0.0010475177550688386, 0.0012340175453573465, 0.006947703659534454, 0.003757019527256489, 0.0176718570291996, 0.0011743978830054402, 0.0036232525017112494, 0.00219246419146657, 0.008196980692446232]]], "attentionHeadNames": ["L9H9", "L9H6", "L10H0"], "tokens": ["<|endoftext|>", "When", " John", " and", " Mary", " went", " to", " the", " shops", ",", " John", " gave", " the", " bag", " to"]}
     )
-    </script></div><div style='max-width: 700px;'><h2>Top 3 Negative Logit Attribution Heads</h2><br/><div id="circuits-vis-6ae2c4e4-98bd" style="margin: 15px 0;"/>
+    </script></div><div style='max-width: 700px;'><h2>Top 3 Negative Logit Attribution Heads</h2><br/><div id="circuits-vis-65209713-881c" style="margin: 15px 0;"/>
     <script crossorigin type="module">
     import { render, AttentionHeads } from "https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js";
     render(
-      "circuits-vis-6ae2c4e4-98bd",
+      "circuits-vis-65209713-881c",
       AttentionHeads,
       {"attention": [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9628916382789612, 0.03710832819342613, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9778335094451904, 0.0034864130429923534, 0.018680110573768616, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8737697601318359, 0.005964328069239855, 0.08126737922430038, 0.03899851813912392, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8745138645172119, 0.017276111990213394, 0.018874678760766983, 0.054268334060907364, 0.035066962242126465, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9704693555831909, 0.0035177830141037703, 0.0003239834331907332, 0.004373463336378336, 0.0003953034174628556, 0.02092011272907257, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7694658041000366, 0.006685878615826368, 0.011966485530138016, 0.040966276079416275, 0.03344443440437317, 0.04401002824306488, 0.0934610366821289, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8177722692489624, 0.009112595580518246, 0.0042336746118962765, 0.03900325670838356, 0.012653318233788013, 0.007337852846831083, 0.07410021126270294, 0.0357869453728199, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8309933543205261, 0.018796611577272415, 0.00330327614210546, 0.01983213610947132, 0.004009497817605734, 0.012078307569026947, 0.03246007487177849, 0.010322270914912224, 0.06820447742938995, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.31403300166130066, 0.01522789802402258, 0.19789263606071472, 0.059379592537879944, 0.3236042559146881, 0.009037697687745094, 0.019917670637369156, 0.012475269846618176, 0.013939663767814636, 0.03449229896068573, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7319568991661072, 0.03087025322020054, 0.005064091179519892, 0.08703826367855072, 0.004982746671885252, 0.044725559651851654, 0.02350614219903946, 0.0063977413810789585, 0.0034696452785283327, 0.048653144389390945, 0.013335512951016426, 0.0, 0.0, 0.0, 0.0], [0.13710583746433258, 0.012017901986837387, 0.14592327177524567, 0.015683427453041077, 0.44125232100486755, 0.004342416767030954, 0.011117888614535332, 0.007466601673513651, 0.0012341630645096302, 0.01946023479104042, 0.16842849552631378, 0.035967420786619186, 0.0, 0.0, 0.0], [0.4052484929561615, 0.009720955975353718, 0.03158125653862953, 0.09247038513422012, 0.048808369785547256, 0.008174203336238861, 0.031708307564258575, 0.02787904627621174, 0.14128026366233826, 0.0302340779453516, 0.03392859548330307, 0.03171432018280029, 0.10725171118974686, 0.0, 0.0], [0.808469295501709, 0.0056702690199017525, 0.006755992770195007, 0.008696584962308407, 0.010473440401256084, 0.003464736510068178, 0.006760997697710991, 0.0025889205280691385, 0.015097705647349358, 0.01603602059185505, 0.011288060806691647, 0.017277495935559273, 0.00779919745400548, 0.07962135970592499, 0.0], [0.02939111366868019, 0.0016060526249930263, 0.06005561351776123, 0.008175727911293507, 0.8099164366722107, 0.00039796155760996044, 0.0028282543644309044, 0.0033632616978138685, 0.0001569116284372285, 0.0036459483671933413, 0.05494074895977974, 0.001666039228439331, 0.015368940308690071, 5.016689101466909e-05, 0.008436810225248337]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9124024510383606, 0.08759760111570358, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.97407466173172, 0.01434413529932499, 0.011581187136471272, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8240247964859009, 0.03700513392686844, 0.06643965095281601, 0.07253041118383408, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5370795726776123, 0.15483324229717255, 0.0903128981590271, 0.12131187319755554, 0.0964624434709549, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9388002157211304, 0.028733234852552414, 0.0003431019140407443, 0.010481251403689384, 0.000839930260553956, 0.02080223336815834, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6318807005882263, 0.059302832931280136, 0.03517911583185196, 0.0317409411072731, 0.1174689382314682, 0.06869616359472275, 0.05573126673698425, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7625216841697693, 0.04435169696807861, 0.010552966967225075, 0.034276288002729416, 0.014203979633748531, 0.012703591957688332, 0.07259687036275864, 0.048792947083711624, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5725991725921631, 0.12681783735752106, 0.03727060556411743, 0.0431414395570755, 0.04129580408334732, 0.046447426080703735, 0.029164383187890053, 0.03751189261674881, 0.06575141847133636, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.12316936254501343, 0.026347072795033455, 0.27960509061813354, 0.041989635676145554, 0.44105827808380127, 0.006138973869383335, 0.007793003693223, 0.007175281643867493, 0.011286810971796513, 0.055436521768569946, 0.0, 0.0, 0.0, 0.0, 0.0], [0.44855940341949463, 0.1432485729455948, 0.010774216614663601, 0.07855041325092316, 0.01404787041246891, 0.014864671975374222, 0.008633698336780071, 0.0030146560166031122, 0.012792127206921577, 0.16408587992191315, 0.10142850130796432, 0.0, 0.0, 0.0, 0.0], [0.10228201746940613, 0.0238832775503397, 0.055250268429517746, 0.0520680733025074, 0.5759841799736023, 0.0011948152678087354, 0.0013662497512996197, 0.0025677781086415052, 0.0018699930515140295, 0.043160032480955124, 0.10887196660041809, 0.03150142729282379, 0.0, 0.0, 0.0], [0.39313194155693054, 0.04406267777085304, 0.04009911045432091, 0.07407406717538834, 0.04061643034219742, 0.007346798665821552, 0.009507199749350548, 0.013865168206393719, 0.0640539675951004, 0.04894229769706726, 0.0627753809094429, 0.0999651551246643, 0.10155977308750153, 0.0, 0.0], [0.7573198676109314, 0.015095721930265427, 0.007350177504122257, 0.013156400062143803, 0.005726401228457689, 0.008556416258215904, 0.008282607421278954, 0.010307075455784798, 0.002904881490394473, 0.017329057678580284, 0.015188980847597122, 0.0796574279665947, 0.030244464054703712, 0.02888045459985733, 0.0], [0.08579524606466293, 0.023773644119501114, 0.028759371489286423, 0.050303805619478226, 0.674644947052002, 0.0010763936443254352, 0.0011514866491779685, 0.002721975091844797, 0.0005617713322862983, 0.039091579616069794, 0.04861969128251076, 0.015520088374614716, 0.0040845321491360664, 0.00043291584006510675, 0.023462524637579918]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9691730737686157, 0.03082684986293316, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9864587187767029, 0.0076598357409238815, 0.00588154187425971, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9318376183509827, 0.011915593408048153, 0.02053167298436165, 0.03571505844593048, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8786704540252686, 0.03427596017718315, 0.01315787062048912, 0.02705124393105507, 0.04684450104832649, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9850097298622131, 0.00501946872100234, 0.0013920071069151163, 0.0018760499078780413, 0.004171107430011034, 0.002531560370698571, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8907089829444885, 0.02026253007352352, 0.020904401317238808, 0.009134626016020775, 0.04057806357741356, 0.003215113654732704, 0.015196278691291809, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9318532347679138, 0.012423796579241753, 0.007052858360111713, 0.00626762630417943, 0.009715708903968334, 0.0013882736675441265, 0.012697475962340832, 0.018601125106215477, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7722940444946289, 0.03829518333077431, 0.03943629562854767, 0.00972924567759037, 0.11019386351108551, 0.009773456491529942, 0.00886810664087534, 0.003977742046117783, 0.007432003039866686, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6659272909164429, 0.019893741235136986, 0.05748209357261658, 0.026046503335237503, 0.1897869110107422, 0.001734035206027329, 0.004116130527108908, 0.004545124247670174, 0.007496052421629429, 0.022972112521529198, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8588331341743469, 0.01852157525718212, 0.010080324485898018, 0.0161746833473444, 0.044801417738199234, 0.018235526978969574, 0.0026932875625789165, 0.0009044647449627519, 0.004338268656283617, 0.01020037941634655, 0.015216901898384094, 0.0, 0.0, 0.0, 0.0], [0.7452806234359741, 0.02059219218790531, 0.033448558300733566, 0.04560310021042824, 0.09397682547569275, 0.0019108005799353123, 0.0039161862805485725, 0.004672198090702295, 0.003216084325686097, 0.0180679801851511, 0.0163005068898201, 0.013014941476285458, 0.0, 0.0, 0.0], [0.7450795769691467, 0.016002941876649857, 0.02564326673746109, 0.03000519424676895, 0.06691146641969681, 0.0032243167515844107, 0.006530732847750187, 0.006865624338388443, 0.04355334863066673, 0.012662945315241814, 0.010547308251261711, 0.010315663181245327, 0.022657567635178566, 0.0, 0.0], [0.8363587260246277, 0.013512525707483292, 0.01655869372189045, 0.008880337700247765, 0.05630703642964363, 0.0022557524498552084, 0.0023984003346413374, 0.0012933476828038692, 0.003150224220007658, 0.013540910556912422, 0.02165023609995842, 0.005392791703343391, 0.00432277237996459, 0.014378219842910767, 0.0], [0.7888551354408264, 0.014087573625147343, 0.022160783410072327, 0.04220619052648544, 0.06532405316829681, 0.0011144893942400813, 0.001956802560016513, 0.002514239400625229, 0.001448699738830328, 0.012026125565171242, 0.009607555344700813, 0.006895523052662611, 0.004557081963866949, 0.003263341262936592, 0.023982450366020203]]], "attentionHeadNames": ["L10H7", "L11H10", "L11H2"], "tokens": ["<|endoftext|>", "When", " John", " and", " Mary", " went", " to", " the", " shops", ",", " John", " gave", " the", " bag", " to"]}
     )
@@ -1306,9 +1306,9 @@ <h2>Residual Stream<a class="headerlink" href="#Residual-Stream" title="Permalin
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area rendered_html docutils container">
-<div>                            <div id="6767b2c2-3357-400a-bf47-8577e0e58602" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("6767b2c2-3357-400a-bf47-8577e0e58602")) {                    Plotly.newPlot(                        "6767b2c2-3357-400a-bf47-8577e0e58602",                        [{"coloraxis":"coloraxis","name":"0","x":["\u003c|endoftext|\u003e_0","When_1"," John_2"," and_3"," Mary_4"," went_5"," to_6"," the_7"," shops_8",",_9"," John_10"," gave_11"," the_12"," bag_13"," to_14"],"z":[[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.00065016746521,-0.00024725322145968676,9.061812306754291e-06,-0.00036435198853723705,-4.832966806134209e-05],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0010517835617065,-2.6816253011929803e-05,-2.0540108380373567e-05,-0.0004592325130943209,-0.0005939850234426558],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0002665519714355,0.0008679538150317967,0.0005159862921573222,-0.0009933760156854987,-0.0008652352844364941],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9949080944061279,0.005429603159427643,0.0016055518062785268,-0.0006179149495437741,-0.0016324687749147415],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9675664305686951,0.03134222328662872,0.0028418514411896467,-0.0012303927214816213,-0.0009862943552434444],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9675208926200867,0.031000729650259018,0.001782458508387208,-0.0004856795712839812,-0.000646778498776257],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9228320121765137,0.05134553834795952,0.004729225765913725,0.0009345413418486714,0.017047081142663956],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.6565485000610352,0.02385673113167286,0.002357447287067771,-1.7318130630883388e-05,0.3186914026737213],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.027302434667944908,0.03142485395073891,0.0018206859240308404,0.0007993190083652735,0.9383869171142578],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.026841893792152405,0.020981015637516975,0.0012513356050476432,0.0003238087520003319,1.0048280954360962],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.005687932018190622,0.014263695105910301,0.0004871227720286697,-8.984619489638135e-05,0.9914218783378601]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Position: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Position"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Logit Difference From Patched Residual Stream"}},                        {"responsive": true}                    ).then(function(){
+<div>                            <div id="ad844fd5-560b-4c3a-bc40-4b48c6c877b8" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("ad844fd5-560b-4c3a-bc40-4b48c6c877b8")) {                    Plotly.newPlot(                        "ad844fd5-560b-4c3a-bc40-4b48c6c877b8",                        [{"coloraxis":"coloraxis","name":"0","x":["\u003c|endoftext|\u003e_0","When_1"," John_2"," and_3"," Mary_4"," went_5"," to_6"," the_7"," shops_8",",_9"," John_10"," gave_11"," the_12"," bag_13"," to_14"],"z":[[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.00065016746521,-0.00024725322145968676,9.061812306754291e-06,-0.00036435198853723705,-4.832966806134209e-05],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0010517835617065,-2.6816253011929803e-05,-2.0540108380373567e-05,-0.0004592325130943209,-0.0005939850234426558],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0002665519714355,0.0008679538150317967,0.0005159862921573222,-0.0009933760156854987,-0.0008652352844364941],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9949080944061279,0.005429603159427643,0.0016055518062785268,-0.0006179149495437741,-0.0016324687749147415],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9675664305686951,0.03134222328662872,0.0028418514411896467,-0.0012303927214816213,-0.0009862943552434444],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9675208926200867,0.031000729650259018,0.001782458508387208,-0.0004856795712839812,-0.000646778498776257],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9228320121765137,0.05134553834795952,0.004729225765913725,0.0009345413418486714,0.017047081142663956],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.6565485000610352,0.02385673113167286,0.002357447287067771,-1.7318130630883388e-05,0.3186914026737213],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.027302434667944908,0.03142485395073891,0.0018206859240308404,0.0007993190083652735,0.9383869171142578],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.026841893792152405,0.020981015637516975,0.0012513356050476432,0.0003238087520003319,1.0048280954360962],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.005687932018190622,0.014263695105910301,0.0004871227720286697,-8.984619489638135e-05,0.9914218783378601]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Position: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Position"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Logit Difference From Patched Residual Stream"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('6767b2c2-3357-400a-bf47-8577e0e58602');
+var gd = document.getElementById('ad844fd5-560b-4c3a-bc40-4b48c6c877b8');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -1394,9 +1394,9 @@ <h2>Layers<a class="headerlink" href="#Layers" title="Permalink to this heading"
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area rendered_html docutils container">
-<div>                            <div id="f4f11dd9-871d-4910-93fd-3680b4af3721" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("f4f11dd9-871d-4910-93fd-3680b4af3721")) {                    Plotly.newPlot(                        "f4f11dd9-871d-4910-93fd-3680b4af3721",                        [{"coloraxis":"coloraxis","name":"0","x":["\u003c|endoftext|\u003e_0","When_1"," John_2"," and_3"," Mary_4"," went_5"," to_6"," the_7"," shops_8",",_9"," John_10"," gave_11"," the_12"," bag_13"," to_14"],"z":[[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.03545692563056946,-0.00024725322145968676,9.061812306754291e-06,-0.00036435198853723705,-4.832966806134209e-05],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.0029837191104888916,7.950901635922492e-05,2.097641845466569e-05,8.054944191826507e-05,-0.0005966364406049252],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.0019127808045595884,0.000666748033836484,0.00039496077806688845,-0.0007048076367937028,-0.0002728612453211099],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.15463140606880188,0.0038022694643586874,0.0005171945667825639,-0.00012015292304567993,-0.0005605234182439744],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.005405941512435675,0.019581804051995277,0.0010072707664221525,-0.00024272232258226722,0.0007941504009068012],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.35209786891937256,0.0010532510932534933,0.00022453156998381019,0.00013320863945409656,8.259674359578639e-05],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.11985944956541061,0.02124394290149212,0.002728075487539172,0.0013410811079666018,0.017973870038986206],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.013310795649886131,0.011509508825838566,0.00037489054375328124,-4.121446545468643e-05,0.2976022958755493],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.001500166254118085,0.017352566123008728,0.0005847889697179198,0.0010113989701494575,0.5697317123413086],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.0001285099278902635,0.006301181390881538,0.00014156564429868013,0.00031266608857549727,0.2715233862400055],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.0009370585321448743,8.679202437633649e-05,0.00033203151542693377,1.543864300401765e-06,-0.1929759979248047],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.4061770737171173]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Position: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Position"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Logit Difference From Patched Attention Layer"}},                        {"responsive": true}                    ).then(function(){
+<div>                            <div id="15943854-8114-4fd6-addb-4e62d14ddf33" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("15943854-8114-4fd6-addb-4e62d14ddf33")) {                    Plotly.newPlot(                        "15943854-8114-4fd6-addb-4e62d14ddf33",                        [{"coloraxis":"coloraxis","name":"0","x":["\u003c|endoftext|\u003e_0","When_1"," John_2"," and_3"," Mary_4"," went_5"," to_6"," the_7"," shops_8",",_9"," John_10"," gave_11"," the_12"," bag_13"," to_14"],"z":[[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.03545692563056946,-0.00024725322145968676,9.061812306754291e-06,-0.00036435198853723705,-4.832966806134209e-05],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.0029837191104888916,7.950901635922492e-05,2.097641845466569e-05,8.054944191826507e-05,-0.0005966364406049252],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.0019127808045595884,0.000666748033836484,0.00039496077806688845,-0.0007048076367937028,-0.0002728612453211099],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.15463140606880188,0.0038022694643586874,0.0005171945667825639,-0.00012015292304567993,-0.0005605234182439744],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.005405941512435675,0.019581804051995277,0.0010072707664221525,-0.00024272232258226722,0.0007941504009068012],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.35209786891937256,0.0010532510932534933,0.00022453156998381019,0.00013320863945409656,8.259674359578639e-05],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.11985944956541061,0.02124394290149212,0.002728075487539172,0.0013410811079666018,0.017973870038986206],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.013310795649886131,0.011509508825838566,0.00037489054375328124,-4.121446545468643e-05,0.2976022958755493],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.001500166254118085,0.017352566123008728,0.0005847889697179198,0.0010113989701494575,0.5697317123413086],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.0001285099278902635,0.006301181390881538,0.00014156564429868013,0.00031266608857549727,0.2715233862400055],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.0009370585321448743,8.679202437633649e-05,0.00033203151542693377,1.543864300401765e-06,-0.1929759979248047],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.4061770737171173]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Position: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Position"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Logit Difference From Patched Attention Layer"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('f4f11dd9-871d-4910-93fd-3680b4af3721');
+var gd = document.getElementById('15943854-8114-4fd6-addb-4e62d14ddf33');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -1443,9 +1443,9 @@ <h2>Layers<a class="headerlink" href="#Layers" title="Permalink to this heading"
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area rendered_html docutils container">
-<div>                            <div id="fa8585f3-ce45-40a0-b698-398c00aca11d" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("fa8585f3-ce45-40a0-b698-398c00aca11d")) {                    Plotly.newPlot(                        "fa8585f3-ce45-40a0-b698-398c00aca11d",                        [{"coloraxis":"coloraxis","name":"0","x":["\u003c|endoftext|\u003e_0","When_1"," John_2"," and_3"," Mary_4"," went_5"," to_6"," the_7"," shops_8",",_9"," John_10"," gave_11"," the_12"," bag_13"," to_14"],"z":[[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.8507904410362244,-0.00027849970501847565,-7.350136729655787e-05,-0.00047316084965132177,3.9704162190901116e-05],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.008864667266607285,0.00022147741401568055,0.00014985552115831524,-4.8866662837099284e-05,0.000304174842312932],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.013550027273595333,5.839834557264112e-05,-0.0003300177922938019,-0.0006387235480360687,0.0007728383643552661],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0019474170403555036,0.0004988359869457781,0.00017297992599196732,0.00016838189912959933,0.0004079157952219248],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.019786769524216652,0.004128897562623024,-4.849747710977681e-05,-0.00016989219875540584,0.0007919353083707392],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.09652478247880936,-0.001882205600850284,-0.00048380010412074625,0.000710244697984308,-0.00018361916590947658],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.015899553894996643,-0.0008504678844474256,0.00012357627565506846,2.8863551051472314e-05,-0.007237736601382494],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.01036097388714552,0.003150892909616232,0.0005310557899065316,0.00023543930728919804,0.008496489375829697],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.012532956898212433,2.2285346858552657e-05,-0.0003539476892910898,8.638927829451859e-05,-0.02163148671388626],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.00033434730721637607,0.0008097903919406235,1.6344823961844668e-05,0.00012938254803884774,0.031624484807252884],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0013600102392956614,-0.0001945940311998129,-9.914294059854001e-05,-0.00014133071817923337,0.02876460924744606],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.020449254661798477]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Position: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Position"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Logit Difference From Patched MLP Layer"}},                        {"responsive": true}                    ).then(function(){
+<div>                            <div id="4a8f65af-b38f-42b0-96a4-9ceb6ed2fec9" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("4a8f65af-b38f-42b0-96a4-9ceb6ed2fec9")) {                    Plotly.newPlot(                        "4a8f65af-b38f-42b0-96a4-9ceb6ed2fec9",                        [{"coloraxis":"coloraxis","name":"0","x":["\u003c|endoftext|\u003e_0","When_1"," John_2"," and_3"," Mary_4"," went_5"," to_6"," the_7"," shops_8",",_9"," John_10"," gave_11"," the_12"," bag_13"," to_14"],"z":[[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.8507904410362244,-0.00027849970501847565,-7.350136729655787e-05,-0.00047316084965132177,3.9704162190901116e-05],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.008864667266607285,0.00022147741401568055,0.00014985552115831524,-4.8866662837099284e-05,0.000304174842312932],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.013550027273595333,5.839834557264112e-05,-0.0003300177922938019,-0.0006387235480360687,0.0007728383643552661],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0019474170403555036,0.0004988359869457781,0.00017297992599196732,0.00016838189912959933,0.0004079157952219248],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.019786769524216652,0.004128897562623024,-4.849747710977681e-05,-0.00016989219875540584,0.0007919353083707392],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.09652478247880936,-0.001882205600850284,-0.00048380010412074625,0.000710244697984308,-0.00018361916590947658],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.015899553894996643,-0.0008504678844474256,0.00012357627565506846,2.8863551051472314e-05,-0.007237736601382494],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.01036097388714552,0.003150892909616232,0.0005310557899065316,0.00023543930728919804,0.008496489375829697],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.012532956898212433,2.2285346858552657e-05,-0.0003539476892910898,8.638927829451859e-05,-0.02163148671388626],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.00033434730721637607,0.0008097903919406235,1.6344823961844668e-05,0.00012938254803884774,0.031624484807252884],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0013600102392956614,-0.0001945940311998129,-9.914294059854001e-05,-0.00014133071817923337,0.02876460924744606],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.020449254661798477]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Position: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Position"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Logit Difference From Patched MLP Layer"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('fa8585f3-ce45-40a0-b698-398c00aca11d');
+var gd = document.getElementById('4a8f65af-b38f-42b0-96a4-9ceb6ed2fec9');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -1526,9 +1526,9 @@ <h2>Heads<a class="headerlink" href="#Heads" title="Permalink to this heading">#
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area rendered_html docutils container">
-<div>                            <div id="a9508ddb-9c9f-4520-a179-20bc68dcb955" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("a9508ddb-9c9f-4520-a179-20bc68dcb955")) {                    Plotly.newPlot(                        "a9508ddb-9c9f-4520-a179-20bc68dcb955",                        [{"coloraxis":"coloraxis","name":"0","z":[[0.000949342327658087,0.016124119982123375,0.0018556242575868964,0.0034389242064207792,-0.009823138825595379,0.011059673503041267,-0.0040638200007379055,-0.0015785341383889318,-0.001208073808811605,0.003828749991953373,-0.004257441032677889,-0.0011428623693063855],[-0.001076644053682685,-0.0003779782564379275,3.0206042538338806e-06,-0.000260510336374864,-0.00014149851631373167,0.0038328110240399837,-0.0004281202855054289,-0.0014291484840214252,-0.0009217541082762182,0.0006938663427717984,0.00043359093251638114,-0.0035708239302039146],[-0.0004966544220224023,0.0008070718613453209,0.0005420306115411222,-0.0005315592279657722,-0.0007152119651436806,-0.0010380810126662254,-0.0009492752142250538,-8.538240945199504e-05,0.0002774928288999945,0.0021076432894915342,-0.0001976146304514259,-0.0016398524167016149],[0.11626167595386505,0.00025017315056174994,-0.0014669395750388503,-0.0003969745012000203,0.01896214485168457,-0.00018882131553255022,0.011170395649969578,-0.0013304082676768303,-0.0007363561890088022,-0.00030310085276141763,-0.00014579448907170445,-0.0002225178322987631],[-0.0016502231592312455,0.00029299859306775033,-0.0014352232683449984,0.030841777101159096,-0.007431894540786743,-0.0002820908557623625,0.006017110776156187,-0.011006947606801987,-0.0012651296565309167,0.001490030437707901,-0.00018016225658357143,0.0029450219590216875],[-0.004210856277495623,0.0029608632903546095,0.002045620232820511,0.0013410139363259077,-0.0012189144035801291,0.34349939227104187,0.0005627049831673503,-0.0001257913827430457,-0.005152345169335604,0.016241183504462242,0.017089638859033585,-0.004173669498413801],[0.039775215089321136,0.015227536670863628,-0.001023011514917016,0.0008081123232841492,-0.004935599863529205,-0.002123551908880472,-0.014273260720074177,0.0013749118661507964,0.0014838549541309476,0.1302708089351654,-0.00033538774005137384,0.0012914760736748576],[0.0003724069392774254,0.0195147804915905,0.00022308839834295213,0.12424899637699127,-0.0004041232750751078,-0.007652163505554199,0.0013012762647122145,-0.0011244030902162194,-0.007449581753462553,0.19224178791046143,-0.0032753082923591137,-0.000501957256346941],[-0.001007740618661046,3.1212908652378246e-05,-0.0008586570620536804,0.012360110878944397,-0.00040429108776152134,-0.004328995477408171,0.31855517625808716,0.002329523442313075,0.002118081087246537,0.0001410957775078714,0.27793484926223755,0.005738543812185526],[0.0058897752314805984,-0.000968372100032866,0.009125681594014168,0.020675498992204666,-0.03700448200106621,0.014263628050684929,-0.04828385263681412,0.05834108963608742,0.0006516450084745884,0.2636081576347351,0.0004918886115774512,-0.0026110773906111717],[0.08374276012182236,0.020676370710134506,-0.0037437702994793653,0.01085111778229475,-0.0010960765648633242,0.0004738992138300091,0.04817944020032883,-0.47990989685058594,0.00018412260396871716,0.011861777864396572,0.06088537722826004,0.0008459034143015742],[0.005328110884875059,-0.011493667028844357,-0.11350765824317932,0.006330112461000681,0.0003166264505125582,-0.0011600126745179296,-0.022669030353426933,0.004070901777595282,0.007316238712519407,-0.008345123380422592,-0.27817434072494507,0.003635196480900049]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Logit Difference From Patched Head Output"}},                        {"responsive": true}                    ).then(function(){
+<div>                            <div id="1555ca46-05e4-4d5a-8182-16f1ca23918a" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("1555ca46-05e4-4d5a-8182-16f1ca23918a")) {                    Plotly.newPlot(                        "1555ca46-05e4-4d5a-8182-16f1ca23918a",                        [{"coloraxis":"coloraxis","name":"0","z":[[0.000949342327658087,0.016124119982123375,0.0018556242575868964,0.0034389242064207792,-0.009823138825595379,0.011059673503041267,-0.0040638200007379055,-0.0015785341383889318,-0.001208073808811605,0.003828749991953373,-0.004257441032677889,-0.0011428623693063855],[-0.001076644053682685,-0.0003779782564379275,3.0206042538338806e-06,-0.000260510336374864,-0.00014149851631373167,0.0038328110240399837,-0.0004281202855054289,-0.0014291484840214252,-0.0009217541082762182,0.0006938663427717984,0.00043359093251638114,-0.0035708239302039146],[-0.0004966544220224023,0.0008070718613453209,0.0005420306115411222,-0.0005315592279657722,-0.0007152119651436806,-0.0010380810126662254,-0.0009492752142250538,-8.538240945199504e-05,0.0002774928288999945,0.0021076432894915342,-0.0001976146304514259,-0.0016398524167016149],[0.11626167595386505,0.00025017315056174994,-0.0014669395750388503,-0.0003969745012000203,0.01896214485168457,-0.00018882131553255022,0.011170395649969578,-0.0013304082676768303,-0.0007363561890088022,-0.00030310085276141763,-0.00014579448907170445,-0.0002225178322987631],[-0.0016502231592312455,0.00029299859306775033,-0.0014352232683449984,0.030841777101159096,-0.007431894540786743,-0.0002820908557623625,0.006017110776156187,-0.011006947606801987,-0.0012651296565309167,0.001490030437707901,-0.00018016225658357143,0.0029450219590216875],[-0.004210856277495623,0.0029608632903546095,0.002045620232820511,0.0013410139363259077,-0.0012189144035801291,0.34349939227104187,0.0005627049831673503,-0.0001257913827430457,-0.005152345169335604,0.016241183504462242,0.017089638859033585,-0.004173669498413801],[0.039775215089321136,0.015227536670863628,-0.001023011514917016,0.0008081123232841492,-0.004935599863529205,-0.002123551908880472,-0.014273260720074177,0.0013749118661507964,0.0014838549541309476,0.1302708089351654,-0.00033538774005137384,0.0012914760736748576],[0.0003724069392774254,0.0195147804915905,0.00022308839834295213,0.12424899637699127,-0.0004041232750751078,-0.007652163505554199,0.0013012762647122145,-0.0011244030902162194,-0.007449581753462553,0.19224178791046143,-0.0032753082923591137,-0.000501957256346941],[-0.001007740618661046,3.1212908652378246e-05,-0.0008586570620536804,0.012360110878944397,-0.00040429108776152134,-0.004328995477408171,0.31855517625808716,0.002329523442313075,0.002118081087246537,0.0001410957775078714,0.27793484926223755,0.005738543812185526],[0.0058897752314805984,-0.000968372100032866,0.009125681594014168,0.020675498992204666,-0.03700448200106621,0.014263628050684929,-0.04828385263681412,0.05834108963608742,0.0006516450084745884,0.2636081576347351,0.0004918886115774512,-0.0026110773906111717],[0.08374276012182236,0.020676370710134506,-0.0037437702994793653,0.01085111778229475,-0.0010960765648633242,0.0004738992138300091,0.04817944020032883,-0.47990989685058594,0.00018412260396871716,0.011861777864396572,0.06088537722826004,0.0008459034143015742],[0.005328110884875059,-0.011493667028844357,-0.11350765824317932,0.006330112461000681,0.0003166264505125582,-0.0011600126745179296,-0.022669030353426933,0.004070901777595282,0.007316238712519407,-0.008345123380422592,-0.27817434072494507,0.003635196480900049]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Logit Difference From Patched Head Output"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('a9508ddb-9c9f-4520-a179-20bc68dcb955');
+var gd = document.getElementById('1555ca46-05e4-4d5a-8182-16f1ca23918a');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -1599,9 +1599,9 @@ <h2>Decomposing Heads<a class="headerlink" href="#Decomposing-Heads" title="Perm
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area rendered_html docutils container">
-<div>                            <div id="d15c92bc-9a5e-459f-a673-8ffed2367454" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("d15c92bc-9a5e-459f-a673-8ffed2367454")) {                    Plotly.newPlot(                        "d15c92bc-9a5e-459f-a673-8ffed2367454",                        [{"coloraxis":"coloraxis","name":"0","z":[[-0.00019808451179414988,0.005339857656508684,0.0006520813331007957,0.0035054110921919346,-0.008983243256807327,0.003481481224298477,-0.0008612077799625695,-3.356226807227358e-05,0.0005169931682758033,0.000442283577285707,-0.003907017409801483,-0.0001884857047116384],[-0.0004399341996759176,-0.00044604254071600735,-6.719166412949562e-05,7.296437252080068e-05,-3.587806349969469e-05,-0.001932314014993608,-0.0001571049797348678,1.550576780573465e-05,0.00020597163529600948,0.0003376699751242995,0.00035139694227837026,-0.0005663632764481008],[0.0002103011793224141,-0.0007198435487225652,0.00048675358993932605,-0.0005971734644845128,-0.0005919041577726603,-0.0005438430234789848,-0.0002275186125189066,-0.00047960481606423855,0.00020617301925085485,0.0011842446401715279,-0.00035743816988542676,-0.0009093025582842529],[0.0010397254955023527,-0.0001205221051350236,-7.69247199059464e-05,-0.0007269251509569585,-0.001310002407990396,-0.002310896525159478,0.010987145826220512,-5.06454634887632e-05,0.00014364650996867567,0.00015129870735108852,-7.920695497887209e-05,-1.9667490050778724e-05],[-0.0005367277772165835,-0.0008130795322358608,-0.000133275767439045,0.030610064044594765,-0.007185581140220165,0.00014941922563593835,0.001333831693045795,-0.011422313749790192,-0.0005329688428901136,0.0005127307958900928,0.00037368229823186994,0.002956030424684286],[7.4508234320092015e-06,6.712453796353657e-06,0.0015982352197170258,0.00033918028930202127,-0.0012586521916091442,-5.423662514658645e-05,0.0006328836898319423,-0.0002699077595025301,7.430685946019366e-05,-0.006704063154757023,0.0031767694745212793,-0.001728456816636026],[0.04863159358501434,0.015315738506615162,-0.0004640990518964827,-0.0001160583269665949,-4.839679240831174e-05,-0.003953501116484404,-0.017375119030475616,-0.0001535809424240142,0.0012203240767121315,-0.00017975950322579592,-0.0004269791825208813,0.0001233748917002231],[-2.849436532414984e-05,-0.001384309260174632,-0.00012176390737295151,0.13321569561958313,-0.00024480317370034754,-0.0073145609349012375,0.0003329376922920346,-0.000794418912846595,-0.007937678135931492,0.20841389894485474,-0.0001913720479933545,-0.0002066764427581802],[-0.0020480367820709944,-0.0003764008288271725,-0.0033146096393465996,-0.009665866382420063,-0.00031719700200483203,-0.005141303408890963,0.3171706795692444,0.002842522691935301,0.00047299303696490824,-0.0011535687372088432,0.27267059683799744,-0.00317572895437479],[-0.00043966571683995426,5.7257228036178276e-05,-0.002063408261165023,0.020066477358341217,-0.007870486006140709,0.01131565310060978,0.003057321300730109,0.06856381893157959,-0.0027472060173749924,-0.009278557263314724,0.0005067567108199,-0.0013152381870895624],[-0.012957250699400902,-0.00304490327835083,-0.017922621220350266,-0.004358463454991579,-0.0011523604625836015,0.0004998764488846064,-0.0031132360454648733,0.01958620175719261,4.3899446609430015e-05,0.012970171868801117,-0.007695660460740328,-0.0009143368806689978],[0.004100503865629435,-0.020459022372961044,-0.035875849425792694,0.014655501581728458,0.0008445944986306131,0.0017805789830163121,-0.018042270094156265,0.003519205143675208,0.008252425119280815,-0.0017664493061602116,0.044166937470436096,0.006474765483289957]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Logit Difference From Patched Head Value"}},                        {"responsive": true}                    ).then(function(){
+<div>                            <div id="c769824a-8892-4cdc-ba58-adc59353a905" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("c769824a-8892-4cdc-ba58-adc59353a905")) {                    Plotly.newPlot(                        "c769824a-8892-4cdc-ba58-adc59353a905",                        [{"coloraxis":"coloraxis","name":"0","z":[[-0.00019808451179414988,0.005339857656508684,0.0006520813331007957,0.0035054110921919346,-0.008983243256807327,0.003481481224298477,-0.0008612077799625695,-3.356226807227358e-05,0.0005169931682758033,0.000442283577285707,-0.003907017409801483,-0.0001884857047116384],[-0.0004399341996759176,-0.00044604254071600735,-6.719166412949562e-05,7.296437252080068e-05,-3.587806349969469e-05,-0.001932314014993608,-0.0001571049797348678,1.550576780573465e-05,0.00020597163529600948,0.0003376699751242995,0.00035139694227837026,-0.0005663632764481008],[0.0002103011793224141,-0.0007198435487225652,0.00048675358993932605,-0.0005971734644845128,-0.0005919041577726603,-0.0005438430234789848,-0.0002275186125189066,-0.00047960481606423855,0.00020617301925085485,0.0011842446401715279,-0.00035743816988542676,-0.0009093025582842529],[0.0010397254955023527,-0.0001205221051350236,-7.69247199059464e-05,-0.0007269251509569585,-0.001310002407990396,-0.002310896525159478,0.010987145826220512,-5.06454634887632e-05,0.00014364650996867567,0.00015129870735108852,-7.920695497887209e-05,-1.9667490050778724e-05],[-0.0005367277772165835,-0.0008130795322358608,-0.000133275767439045,0.030610064044594765,-0.007185581140220165,0.00014941922563593835,0.001333831693045795,-0.011422313749790192,-0.0005329688428901136,0.0005127307958900928,0.00037368229823186994,0.002956030424684286],[7.4508234320092015e-06,6.712453796353657e-06,0.0015982352197170258,0.00033918028930202127,-0.0012586521916091442,-5.423662514658645e-05,0.0006328836898319423,-0.0002699077595025301,7.430685946019366e-05,-0.006704063154757023,0.0031767694745212793,-0.001728456816636026],[0.04863159358501434,0.015315738506615162,-0.0004640990518964827,-0.0001160583269665949,-4.839679240831174e-05,-0.003953501116484404,-0.017375119030475616,-0.0001535809424240142,0.0012203240767121315,-0.00017975950322579592,-0.0004269791825208813,0.0001233748917002231],[-2.849436532414984e-05,-0.001384309260174632,-0.00012176390737295151,0.13321569561958313,-0.00024480317370034754,-0.0073145609349012375,0.0003329376922920346,-0.000794418912846595,-0.007937678135931492,0.20841389894485474,-0.0001913720479933545,-0.0002066764427581802],[-0.0020480367820709944,-0.0003764008288271725,-0.0033146096393465996,-0.009665866382420063,-0.00031719700200483203,-0.005141303408890963,0.3171706795692444,0.002842522691935301,0.00047299303696490824,-0.0011535687372088432,0.27267059683799744,-0.00317572895437479],[-0.00043966571683995426,5.7257228036178276e-05,-0.002063408261165023,0.020066477358341217,-0.007870486006140709,0.01131565310060978,0.003057321300730109,0.06856381893157959,-0.0027472060173749924,-0.009278557263314724,0.0005067567108199,-0.0013152381870895624],[-0.012957250699400902,-0.00304490327835083,-0.017922621220350266,-0.004358463454991579,-0.0011523604625836015,0.0004998764488846064,-0.0031132360454648733,0.01958620175719261,4.3899446609430015e-05,0.012970171868801117,-0.007695660460740328,-0.0009143368806689978],[0.004100503865629435,-0.020459022372961044,-0.035875849425792694,0.014655501581728458,0.0008445944986306131,0.0017805789830163121,-0.018042270094156265,0.003519205143675208,0.008252425119280815,-0.0017664493061602116,0.044166937470436096,0.006474765483289957]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Logit Difference From Patched Head Value"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('d15c92bc-9a5e-459f-a673-8ffed2367454');
+var gd = document.getElementById('c769824a-8892-4cdc-ba58-adc59353a905');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -1655,9 +1655,9 @@ <h2>Decomposing Heads<a class="headerlink" href="#Decomposing-Heads" title="Perm
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area rendered_html docutils container">
-<div>                            <div id="01dae98a-ebc8-4e3b-adb0-ff57e3b90a6d" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("01dae98a-ebc8-4e3b-adb0-ff57e3b90a6d")) {                    Plotly.newPlot(                        "01dae98a-ebc8-4e3b-adb0-ff57e3b90a6d",                        [{"hovertemplate":"\u003cb\u003e%{hovertext}\u003c\u002fb\u003e\u003cbr\u003e\u003cbr\u003eValue Patch=%{x}\u003cbr\u003eOutput Patch=%{y}\u003cbr\u003eLayer=%{marker.color}\u003cextra\u003e\u003c\u002fextra\u003e","hovertext":["L0H0","L0H1","L0H2","L0H3","L0H4","L0H5","L0H6","L0H7","L0H8","L0H9","L0H10","L0H11","L1H0","L1H1","L1H2","L1H3","L1H4","L1H5","L1H6","L1H7","L1H8","L1H9","L1H10","L1H11","L2H0","L2H1","L2H2","L2H3","L2H4","L2H5","L2H6","L2H7","L2H8","L2H9","L2H10","L2H11","L3H0","L3H1","L3H2","L3H3","L3H4","L3H5","L3H6","L3H7","L3H8","L3H9","L3H10","L3H11","L4H0","L4H1","L4H2","L4H3","L4H4","L4H5","L4H6","L4H7","L4H8","L4H9","L4H10","L4H11","L5H0","L5H1","L5H2","L5H3","L5H4","L5H5","L5H6","L5H7","L5H8","L5H9","L5H10","L5H11","L6H0","L6H1","L6H2","L6H3","L6H4","L6H5","L6H6","L6H7","L6H8","L6H9","L6H10","L6H11","L7H0","L7H1","L7H2","L7H3","L7H4","L7H5","L7H6","L7H7","L7H8","L7H9","L7H10","L7H11","L8H0","L8H1","L8H2","L8H3","L8H4","L8H5","L8H6","L8H7","L8H8","L8H9","L8H10","L8H11","L9H0","L9H1","L9H2","L9H3","L9H4","L9H5","L9H6","L9H7","L9H8","L9H9","L9H10","L9H11","L10H0","L10H1","L10H2","L10H3","L10H4","L10H5","L10H6","L10H7","L10H8","L10H9","L10H10","L10H11","L11H0","L11H1","L11H2","L11H3","L11H4","L11H5","L11H6","L11H7","L11H8","L11H9","L11H10","L11H11"],"legendgroup":"","marker":{"color":[0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4,4,4,4,5,5,5,5,5,5,5,5,5,5,5,5,6,6,6,6,6,6,6,6,6,6,6,6,7,7,7,7,7,7,7,7,7,7,7,7,8,8,8,8,8,8,8,8,8,8,8,8,9,9,9,9,9,9,9,9,9,9,9,9,10,10,10,10,10,10,10,10,10,10,10,10,11,11,11,11,11,11,11,11,11,11,11,11],"coloraxis":"coloraxis","symbol":"circle"},"mode":"markers","name":"","orientation":"v","showlegend":false,"x":[-0.00019808451179414988,0.005339857656508684,0.0006520813331007957,0.0035054110921919346,-0.008983243256807327,0.003481481224298477,-0.0008612077799625695,-3.356226807227358e-05,0.0005169931682758033,0.000442283577285707,-0.003907017409801483,-0.0001884857047116384,-0.0004399341996759176,-0.00044604254071600735,-6.719166412949562e-05,7.296437252080068e-05,-3.587806349969469e-05,-0.001932314014993608,-0.0001571049797348678,1.550576780573465e-05,0.00020597163529600948,0.0003376699751242995,0.00035139694227837026,-0.0005663632764481008,0.0002103011793224141,-0.0007198435487225652,0.00048675358993932605,-0.0005971734644845128,-0.0005919041577726603,-0.0005438430234789848,-0.0002275186125189066,-0.00047960481606423855,0.00020617301925085485,0.0011842446401715279,-0.00035743816988542676,-0.0009093025582842529,0.0010397254955023527,-0.0001205221051350236,-7.69247199059464e-05,-0.0007269251509569585,-0.001310002407990396,-0.002310896525159478,0.010987145826220512,-5.06454634887632e-05,0.00014364650996867567,0.00015129870735108852,-7.920695497887209e-05,-1.9667490050778724e-05,-0.0005367277772165835,-0.0008130795322358608,-0.000133275767439045,0.030610064044594765,-0.007185581140220165,0.00014941922563593835,0.001333831693045795,-0.011422313749790192,-0.0005329688428901136,0.0005127307958900928,0.00037368229823186994,0.002956030424684286,7.4508234320092015e-06,6.712453796353657e-06,0.0015982352197170258,0.00033918028930202127,-0.0012586521916091442,-5.423662514658645e-05,0.0006328836898319423,-0.0002699077595025301,7.430685946019366e-05,-0.006704063154757023,0.0031767694745212793,-0.001728456816636026,0.04863159358501434,0.015315738506615162,-0.0004640990518964827,-0.0001160583269665949,-4.839679240831174e-05,-0.003953501116484404,-0.017375119030475616,-0.0001535809424240142,0.0012203240767121315,-0.00017975950322579592,-0.0004269791825208813,0.0001233748917002231,-2.849436532414984e-05,-0.001384309260174632,-0.00012176390737295151,0.13321569561958313,-0.00024480317370034754,-0.0073145609349012375,0.0003329376922920346,-0.000794418912846595,-0.007937678135931492,0.20841389894485474,-0.0001913720479933545,-0.0002066764427581802,-0.0020480367820709944,-0.0003764008288271725,-0.0033146096393465996,-0.009665866382420063,-0.00031719700200483203,-0.005141303408890963,0.3171706795692444,0.002842522691935301,0.00047299303696490824,-0.0011535687372088432,0.27267059683799744,-0.00317572895437479,-0.00043966571683995426,5.7257228036178276e-05,-0.002063408261165023,0.020066477358341217,-0.007870486006140709,0.01131565310060978,0.003057321300730109,0.06856381893157959,-0.0027472060173749924,-0.009278557263314724,0.0005067567108199,-0.0013152381870895624,-0.012957250699400902,-0.00304490327835083,-0.017922621220350266,-0.004358463454991579,-0.0011523604625836015,0.0004998764488846064,-0.0031132360454648733,0.01958620175719261,4.3899446609430015e-05,0.012970171868801117,-0.007695660460740328,-0.0009143368806689978,0.004100503865629435,-0.020459022372961044,-0.035875849425792694,0.014655501581728458,0.0008445944986306131,0.0017805789830163121,-0.018042270094156265,0.003519205143675208,0.008252425119280815,-0.0017664493061602116,0.044166937470436096,0.006474765483289957],"xaxis":"x","y":[0.000949342327658087,0.016124119982123375,0.0018556242575868964,0.0034389242064207792,-0.009823138825595379,0.011059673503041267,-0.0040638200007379055,-0.0015785341383889318,-0.001208073808811605,0.003828749991953373,-0.004257441032677889,-0.0011428623693063855,-0.001076644053682685,-0.0003779782564379275,3.0206042538338806e-06,-0.000260510336374864,-0.00014149851631373167,0.0038328110240399837,-0.0004281202855054289,-0.0014291484840214252,-0.0009217541082762182,0.0006938663427717984,0.00043359093251638114,-0.0035708239302039146,-0.0004966544220224023,0.0008070718613453209,0.0005420306115411222,-0.0005315592279657722,-0.0007152119651436806,-0.0010380810126662254,-0.0009492752142250538,-8.538240945199504e-05,0.0002774928288999945,0.0021076432894915342,-0.0001976146304514259,-0.0016398524167016149,0.11626167595386505,0.00025017315056174994,-0.0014669395750388503,-0.0003969745012000203,0.01896214485168457,-0.00018882131553255022,0.011170395649969578,-0.0013304082676768303,-0.0007363561890088022,-0.00030310085276141763,-0.00014579448907170445,-0.0002225178322987631,-0.0016502231592312455,0.00029299859306775033,-0.0014352232683449984,0.030841777101159096,-0.007431894540786743,-0.0002820908557623625,0.006017110776156187,-0.011006947606801987,-0.0012651296565309167,0.001490030437707901,-0.00018016225658357143,0.0029450219590216875,-0.004210856277495623,0.0029608632903546095,0.002045620232820511,0.0013410139363259077,-0.0012189144035801291,0.34349939227104187,0.0005627049831673503,-0.0001257913827430457,-0.005152345169335604,0.016241183504462242,0.017089638859033585,-0.004173669498413801,0.039775215089321136,0.015227536670863628,-0.001023011514917016,0.0008081123232841492,-0.004935599863529205,-0.002123551908880472,-0.014273260720074177,0.0013749118661507964,0.0014838549541309476,0.1302708089351654,-0.00033538774005137384,0.0012914760736748576,0.0003724069392774254,0.0195147804915905,0.00022308839834295213,0.12424899637699127,-0.0004041232750751078,-0.007652163505554199,0.0013012762647122145,-0.0011244030902162194,-0.007449581753462553,0.19224178791046143,-0.0032753082923591137,-0.000501957256346941,-0.001007740618661046,3.1212908652378246e-05,-0.0008586570620536804,0.012360110878944397,-0.00040429108776152134,-0.004328995477408171,0.31855517625808716,0.002329523442313075,0.002118081087246537,0.0001410957775078714,0.27793484926223755,0.005738543812185526,0.0058897752314805984,-0.000968372100032866,0.009125681594014168,0.020675498992204666,-0.03700448200106621,0.014263628050684929,-0.04828385263681412,0.05834108963608742,0.0006516450084745884,0.2636081576347351,0.0004918886115774512,-0.0026110773906111717,0.08374276012182236,0.020676370710134506,-0.0037437702994793653,0.01085111778229475,-0.0010960765648633242,0.0004738992138300091,0.04817944020032883,-0.47990989685058594,0.00018412260396871716,0.011861777864396572,0.06088537722826004,0.0008459034143015742,0.005328110884875059,-0.011493667028844357,-0.11350765824317932,0.006330112461000681,0.0003166264505125582,-0.0011600126745179296,-0.022669030353426933,0.004070901777595282,0.007316238712519407,-0.008345123380422592,-0.27817434072494507,0.003635196480900049],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"Value Patch"},"range":[-0.5,0.5]},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"Output Patch"},"range":[-0.5,0.5]},"coloraxis":{"colorbar":{"title":{"text":"Layer"}},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]},"legend":{"tracegroupgap":0},"title":{"text":"Scatter plot of output patching vs value patching"}},                        {"responsive": true}                    ).then(function(){
+<div>                            <div id="1c9555de-e7b8-41d2-973b-2deffd53bf8d" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("1c9555de-e7b8-41d2-973b-2deffd53bf8d")) {                    Plotly.newPlot(                        "1c9555de-e7b8-41d2-973b-2deffd53bf8d",                        [{"hovertemplate":"\u003cb\u003e%{hovertext}\u003c\u002fb\u003e\u003cbr\u003e\u003cbr\u003eValue Patch=%{x}\u003cbr\u003eOutput Patch=%{y}\u003cbr\u003eLayer=%{marker.color}\u003cextra\u003e\u003c\u002fextra\u003e","hovertext":["L0H0","L0H1","L0H2","L0H3","L0H4","L0H5","L0H6","L0H7","L0H8","L0H9","L0H10","L0H11","L1H0","L1H1","L1H2","L1H3","L1H4","L1H5","L1H6","L1H7","L1H8","L1H9","L1H10","L1H11","L2H0","L2H1","L2H2","L2H3","L2H4","L2H5","L2H6","L2H7","L2H8","L2H9","L2H10","L2H11","L3H0","L3H1","L3H2","L3H3","L3H4","L3H5","L3H6","L3H7","L3H8","L3H9","L3H10","L3H11","L4H0","L4H1","L4H2","L4H3","L4H4","L4H5","L4H6","L4H7","L4H8","L4H9","L4H10","L4H11","L5H0","L5H1","L5H2","L5H3","L5H4","L5H5","L5H6","L5H7","L5H8","L5H9","L5H10","L5H11","L6H0","L6H1","L6H2","L6H3","L6H4","L6H5","L6H6","L6H7","L6H8","L6H9","L6H10","L6H11","L7H0","L7H1","L7H2","L7H3","L7H4","L7H5","L7H6","L7H7","L7H8","L7H9","L7H10","L7H11","L8H0","L8H1","L8H2","L8H3","L8H4","L8H5","L8H6","L8H7","L8H8","L8H9","L8H10","L8H11","L9H0","L9H1","L9H2","L9H3","L9H4","L9H5","L9H6","L9H7","L9H8","L9H9","L9H10","L9H11","L10H0","L10H1","L10H2","L10H3","L10H4","L10H5","L10H6","L10H7","L10H8","L10H9","L10H10","L10H11","L11H0","L11H1","L11H2","L11H3","L11H4","L11H5","L11H6","L11H7","L11H8","L11H9","L11H10","L11H11"],"legendgroup":"","marker":{"color":[0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4,4,4,4,5,5,5,5,5,5,5,5,5,5,5,5,6,6,6,6,6,6,6,6,6,6,6,6,7,7,7,7,7,7,7,7,7,7,7,7,8,8,8,8,8,8,8,8,8,8,8,8,9,9,9,9,9,9,9,9,9,9,9,9,10,10,10,10,10,10,10,10,10,10,10,10,11,11,11,11,11,11,11,11,11,11,11,11],"coloraxis":"coloraxis","symbol":"circle"},"mode":"markers","name":"","orientation":"v","showlegend":false,"x":[-0.00019808451179414988,0.005339857656508684,0.0006520813331007957,0.0035054110921919346,-0.008983243256807327,0.003481481224298477,-0.0008612077799625695,-3.356226807227358e-05,0.0005169931682758033,0.000442283577285707,-0.003907017409801483,-0.0001884857047116384,-0.0004399341996759176,-0.00044604254071600735,-6.719166412949562e-05,7.296437252080068e-05,-3.587806349969469e-05,-0.001932314014993608,-0.0001571049797348678,1.550576780573465e-05,0.00020597163529600948,0.0003376699751242995,0.00035139694227837026,-0.0005663632764481008,0.0002103011793224141,-0.0007198435487225652,0.00048675358993932605,-0.0005971734644845128,-0.0005919041577726603,-0.0005438430234789848,-0.0002275186125189066,-0.00047960481606423855,0.00020617301925085485,0.0011842446401715279,-0.00035743816988542676,-0.0009093025582842529,0.0010397254955023527,-0.0001205221051350236,-7.69247199059464e-05,-0.0007269251509569585,-0.001310002407990396,-0.002310896525159478,0.010987145826220512,-5.06454634887632e-05,0.00014364650996867567,0.00015129870735108852,-7.920695497887209e-05,-1.9667490050778724e-05,-0.0005367277772165835,-0.0008130795322358608,-0.000133275767439045,0.030610064044594765,-0.007185581140220165,0.00014941922563593835,0.001333831693045795,-0.011422313749790192,-0.0005329688428901136,0.0005127307958900928,0.00037368229823186994,0.002956030424684286,7.4508234320092015e-06,6.712453796353657e-06,0.0015982352197170258,0.00033918028930202127,-0.0012586521916091442,-5.423662514658645e-05,0.0006328836898319423,-0.0002699077595025301,7.430685946019366e-05,-0.006704063154757023,0.0031767694745212793,-0.001728456816636026,0.04863159358501434,0.015315738506615162,-0.0004640990518964827,-0.0001160583269665949,-4.839679240831174e-05,-0.003953501116484404,-0.017375119030475616,-0.0001535809424240142,0.0012203240767121315,-0.00017975950322579592,-0.0004269791825208813,0.0001233748917002231,-2.849436532414984e-05,-0.001384309260174632,-0.00012176390737295151,0.13321569561958313,-0.00024480317370034754,-0.0073145609349012375,0.0003329376922920346,-0.000794418912846595,-0.007937678135931492,0.20841389894485474,-0.0001913720479933545,-0.0002066764427581802,-0.0020480367820709944,-0.0003764008288271725,-0.0033146096393465996,-0.009665866382420063,-0.00031719700200483203,-0.005141303408890963,0.3171706795692444,0.002842522691935301,0.00047299303696490824,-0.0011535687372088432,0.27267059683799744,-0.00317572895437479,-0.00043966571683995426,5.7257228036178276e-05,-0.002063408261165023,0.020066477358341217,-0.007870486006140709,0.01131565310060978,0.003057321300730109,0.06856381893157959,-0.0027472060173749924,-0.009278557263314724,0.0005067567108199,-0.0013152381870895624,-0.012957250699400902,-0.00304490327835083,-0.017922621220350266,-0.004358463454991579,-0.0011523604625836015,0.0004998764488846064,-0.0031132360454648733,0.01958620175719261,4.3899446609430015e-05,0.012970171868801117,-0.007695660460740328,-0.0009143368806689978,0.004100503865629435,-0.020459022372961044,-0.035875849425792694,0.014655501581728458,0.0008445944986306131,0.0017805789830163121,-0.018042270094156265,0.003519205143675208,0.008252425119280815,-0.0017664493061602116,0.044166937470436096,0.006474765483289957],"xaxis":"x","y":[0.000949342327658087,0.016124119982123375,0.0018556242575868964,0.0034389242064207792,-0.009823138825595379,0.011059673503041267,-0.0040638200007379055,-0.0015785341383889318,-0.001208073808811605,0.003828749991953373,-0.004257441032677889,-0.0011428623693063855,-0.001076644053682685,-0.0003779782564379275,3.0206042538338806e-06,-0.000260510336374864,-0.00014149851631373167,0.0038328110240399837,-0.0004281202855054289,-0.0014291484840214252,-0.0009217541082762182,0.0006938663427717984,0.00043359093251638114,-0.0035708239302039146,-0.0004966544220224023,0.0008070718613453209,0.0005420306115411222,-0.0005315592279657722,-0.0007152119651436806,-0.0010380810126662254,-0.0009492752142250538,-8.538240945199504e-05,0.0002774928288999945,0.0021076432894915342,-0.0001976146304514259,-0.0016398524167016149,0.11626167595386505,0.00025017315056174994,-0.0014669395750388503,-0.0003969745012000203,0.01896214485168457,-0.00018882131553255022,0.011170395649969578,-0.0013304082676768303,-0.0007363561890088022,-0.00030310085276141763,-0.00014579448907170445,-0.0002225178322987631,-0.0016502231592312455,0.00029299859306775033,-0.0014352232683449984,0.030841777101159096,-0.007431894540786743,-0.0002820908557623625,0.006017110776156187,-0.011006947606801987,-0.0012651296565309167,0.001490030437707901,-0.00018016225658357143,0.0029450219590216875,-0.004210856277495623,0.0029608632903546095,0.002045620232820511,0.0013410139363259077,-0.0012189144035801291,0.34349939227104187,0.0005627049831673503,-0.0001257913827430457,-0.005152345169335604,0.016241183504462242,0.017089638859033585,-0.004173669498413801,0.039775215089321136,0.015227536670863628,-0.001023011514917016,0.0008081123232841492,-0.004935599863529205,-0.002123551908880472,-0.014273260720074177,0.0013749118661507964,0.0014838549541309476,0.1302708089351654,-0.00033538774005137384,0.0012914760736748576,0.0003724069392774254,0.0195147804915905,0.00022308839834295213,0.12424899637699127,-0.0004041232750751078,-0.007652163505554199,0.0013012762647122145,-0.0011244030902162194,-0.007449581753462553,0.19224178791046143,-0.0032753082923591137,-0.000501957256346941,-0.001007740618661046,3.1212908652378246e-05,-0.0008586570620536804,0.012360110878944397,-0.00040429108776152134,-0.004328995477408171,0.31855517625808716,0.002329523442313075,0.002118081087246537,0.0001410957775078714,0.27793484926223755,0.005738543812185526,0.0058897752314805984,-0.000968372100032866,0.009125681594014168,0.020675498992204666,-0.03700448200106621,0.014263628050684929,-0.04828385263681412,0.05834108963608742,0.0006516450084745884,0.2636081576347351,0.0004918886115774512,-0.0026110773906111717,0.08374276012182236,0.020676370710134506,-0.0037437702994793653,0.01085111778229475,-0.0010960765648633242,0.0004738992138300091,0.04817944020032883,-0.47990989685058594,0.00018412260396871716,0.011861777864396572,0.06088537722826004,0.0008459034143015742,0.005328110884875059,-0.011493667028844357,-0.11350765824317932,0.006330112461000681,0.0003166264505125582,-0.0011600126745179296,-0.022669030353426933,0.004070901777595282,0.007316238712519407,-0.008345123380422592,-0.27817434072494507,0.003635196480900049],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"Value Patch"},"range":[-0.5,0.5]},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"Output Patch"},"range":[-0.5,0.5]},"coloraxis":{"colorbar":{"title":{"text":"Layer"}},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]},"legend":{"tracegroupgap":0},"title":{"text":"Scatter plot of output patching vs value patching"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('01dae98a-ebc8-4e3b-adb0-ff57e3b90a6d');
+var gd = document.getElementById('1c9555de-e7b8-41d2-973b-2deffd53bf8d');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -1744,9 +1744,9 @@ <h2>Decomposing Heads<a class="headerlink" href="#Decomposing-Heads" title="Perm
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area rendered_html docutils container">
-<div>                            <div id="19f79480-4a62-4c7c-977d-3dc9f4e33aff" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("19f79480-4a62-4c7c-977d-3dc9f4e33aff")) {                    Plotly.newPlot(                        "19f79480-4a62-4c7c-977d-3dc9f4e33aff",                        [{"coloraxis":"coloraxis","name":"0","z":[[0.0006398981786333025,0.005319283809512854,0.0011588380439206958,-5.923740172875114e-05,-0.0010669109178707004,0.005080018658190966,-0.0030812176410108805,-0.0020524668507277966,-0.0014406939735636115,0.003492556745186448,-0.0025678491219878197,-0.0009152094717137516],[-0.0007601853576488793,0.0001696908293524757,0.00012250228610355407,-0.00034918184974230826,1.5405081285280176e-05,0.005008597858250141,-0.0002970260684378445,-0.0014443857362493873,-0.001099835499189794,0.00047436909517273307,5.108177356305532e-05,-0.003494771895930171],[-0.0007234682561829686,0.0017475537024438381,-0.0001551919267512858,5.749216506956145e-05,-9.699495421955362e-05,-0.00042355581535957754,-0.000791196885984391,0.0002723913639783859,0.00010212998313363642,0.00042187771759927273,0.00015183570212684572,-0.0007432028651237488],[0.11458060890436172,0.00021201284835115075,-0.000942025741096586,0.0004294627869967371,0.02004331909120083,0.002104991814121604,7.662265852559358e-05,-0.0015435622772201896,-0.0008476486545987427,-0.0005817683413624763,0.00011934742360608652,-1.9130493456032127e-05],[-0.001126819639466703,0.0012377764796838164,-0.0012320708483457565,-0.0005953610525466502,-0.0007538085337728262,-0.0005843526450917125,0.004813097883015871,0.00018163899949286133,-0.0005363250384107232,0.0008580865105614066,-0.00029840212664566934,-1.2183103535789996e-05],[-0.004240894690155983,0.0029526741709560156,0.0005220275488682091,0.0009532691328786314,0.0001625084987608716,0.3435087502002716,-0.0003038727736566216,0.00010364028275944293,-0.005300321150571108,0.024866117164492607,0.014383613131940365,-0.0023283488117158413],[-0.0023879888467490673,-0.002172317821532488,-0.00047651707427576184,0.0004340272571425885,-0.004673780873417854,0.001858678413555026,-0.0026543056592345238,0.0014364985981956124,0.000303067296044901,0.1304299533367157,8.957769750850275e-05,0.0011777335312217474],[0.0003194456803612411,0.020570717751979828,0.00031887509976513684,-0.002512404229491949,-0.0002621884341351688,-0.00024587716325186193,0.0005522335413843393,-0.00043134228326380253,0.0002570534124970436,0.008090788498520851,-0.003069269470870495,-0.0004241599526721984],[0.0009768297895789146,0.00039294702582992613,0.0017541655106469989,0.022596266120672226,-4.443644138518721e-05,0.00014159921556711197,0.00958488043397665,-0.0003154182049911469,0.0015266804257407784,0.0011816268088296056,-0.01077402476221323,0.009365785866975784],[0.006314304657280445,-0.001094431965611875,0.011661848053336143,0.0013478941982612014,-0.02918611839413643,0.0038340529426932335,-0.04409320279955864,-0.005031823180615902,0.004822427872568369,0.2766473889350891,-3.1280032999347895e-05,-0.0006619150517508388],[0.09538879990577698,0.025069067254662514,0.014238725416362286,0.01475451048463583,9.914294059854001e-05,-8.977906691143289e-05,0.0508294515311718,-0.5050979852676392,0.00014717054727952927,-0.0016020277980715036,0.06883121281862259,0.0023271406535059214],[0.0013427927624434233,0.009631095454096794,-0.07776330411434174,-0.007728920318186283,-0.0005721024353988469,-0.0029568022582679987,-0.00494667561724782,0.000458796217571944,-0.0006325145368464291,-0.006519974209368229,-0.32048866152763367,-0.002472163178026676]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Logit Difference From Patched Head Pattern"}},                        {"responsive": true}                    ).then(function(){
+<div>                            <div id="ffed76f4-77df-4501-b30f-4cced68f4dbf" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("ffed76f4-77df-4501-b30f-4cced68f4dbf")) {                    Plotly.newPlot(                        "ffed76f4-77df-4501-b30f-4cced68f4dbf",                        [{"coloraxis":"coloraxis","name":"0","z":[[0.0006398981786333025,0.005319283809512854,0.0011588380439206958,-5.923740172875114e-05,-0.0010669109178707004,0.005080018658190966,-0.0030812176410108805,-0.0020524668507277966,-0.0014406939735636115,0.003492556745186448,-0.0025678491219878197,-0.0009152094717137516],[-0.0007601853576488793,0.0001696908293524757,0.00012250228610355407,-0.00034918184974230826,1.5405081285280176e-05,0.005008597858250141,-0.0002970260684378445,-0.0014443857362493873,-0.001099835499189794,0.00047436909517273307,5.108177356305532e-05,-0.003494771895930171],[-0.0007234682561829686,0.0017475537024438381,-0.0001551919267512858,5.749216506956145e-05,-9.699495421955362e-05,-0.00042355581535957754,-0.000791196885984391,0.0002723913639783859,0.00010212998313363642,0.00042187771759927273,0.00015183570212684572,-0.0007432028651237488],[0.11458060890436172,0.00021201284835115075,-0.000942025741096586,0.0004294627869967371,0.02004331909120083,0.002104991814121604,7.662265852559358e-05,-0.0015435622772201896,-0.0008476486545987427,-0.0005817683413624763,0.00011934742360608652,-1.9130493456032127e-05],[-0.001126819639466703,0.0012377764796838164,-0.0012320708483457565,-0.0005953610525466502,-0.0007538085337728262,-0.0005843526450917125,0.004813097883015871,0.00018163899949286133,-0.0005363250384107232,0.0008580865105614066,-0.00029840212664566934,-1.2183103535789996e-05],[-0.004240894690155983,0.0029526741709560156,0.0005220275488682091,0.0009532691328786314,0.0001625084987608716,0.3435087502002716,-0.0003038727736566216,0.00010364028275944293,-0.005300321150571108,0.024866117164492607,0.014383613131940365,-0.0023283488117158413],[-0.0023879888467490673,-0.002172317821532488,-0.00047651707427576184,0.0004340272571425885,-0.004673780873417854,0.001858678413555026,-0.0026543056592345238,0.0014364985981956124,0.000303067296044901,0.1304299533367157,8.957769750850275e-05,0.0011777335312217474],[0.0003194456803612411,0.020570717751979828,0.00031887509976513684,-0.002512404229491949,-0.0002621884341351688,-0.00024587716325186193,0.0005522335413843393,-0.00043134228326380253,0.0002570534124970436,0.008090788498520851,-0.003069269470870495,-0.0004241599526721984],[0.0009768297895789146,0.00039294702582992613,0.0017541655106469989,0.022596266120672226,-4.443644138518721e-05,0.00014159921556711197,0.00958488043397665,-0.0003154182049911469,0.0015266804257407784,0.0011816268088296056,-0.01077402476221323,0.009365785866975784],[0.006314304657280445,-0.001094431965611875,0.011661848053336143,0.0013478941982612014,-0.02918611839413643,0.0038340529426932335,-0.04409320279955864,-0.005031823180615902,0.004822427872568369,0.2766473889350891,-3.1280032999347895e-05,-0.0006619150517508388],[0.09538879990577698,0.025069067254662514,0.014238725416362286,0.01475451048463583,9.914294059854001e-05,-8.977906691143289e-05,0.0508294515311718,-0.5050979852676392,0.00014717054727952927,-0.0016020277980715036,0.06883121281862259,0.0023271406535059214],[0.0013427927624434233,0.009631095454096794,-0.07776330411434174,-0.007728920318186283,-0.0005721024353988469,-0.0029568022582679987,-0.00494667561724782,0.000458796217571944,-0.0006325145368464291,-0.006519974209368229,-0.32048866152763367,-0.002472163178026676]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Logit Difference From Patched Head Pattern"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('19f79480-4a62-4c7c-977d-3dc9f4e33aff');
+var gd = document.getElementById('ffed76f4-77df-4501-b30f-4cced68f4dbf');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -1774,9 +1774,9 @@ <h2>Decomposing Heads<a class="headerlink" href="#Decomposing-Heads" title="Perm
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area rendered_html docutils container">
-<div>                            <div id="f93bc8e7-172e-489a-86d5-ed9f1696b43b" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("f93bc8e7-172e-489a-86d5-ed9f1696b43b")) {                    Plotly.newPlot(                        "f93bc8e7-172e-489a-86d5-ed9f1696b43b",                        [{"hovertemplate":"\u003cb\u003e%{hovertext}\u003c\u002fb\u003e\u003cbr\u003e\u003cbr\u003eAttention Patch=%{x}\u003cbr\u003eOutput Patch=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","hovertext":["L0H0","L0H1","L0H2","L0H3","L0H4","L0H5","L0H6","L0H7","L0H8","L0H9","L0H10","L0H11","L1H0","L1H1","L1H2","L1H3","L1H4","L1H5","L1H6","L1H7","L1H8","L1H9","L1H10","L1H11","L2H0","L2H1","L2H2","L2H3","L2H4","L2H5","L2H6","L2H7","L2H8","L2H9","L2H10","L2H11","L3H0","L3H1","L3H2","L3H3","L3H4","L3H5","L3H6","L3H7","L3H8","L3H9","L3H10","L3H11","L4H0","L4H1","L4H2","L4H3","L4H4","L4H5","L4H6","L4H7","L4H8","L4H9","L4H10","L4H11","L5H0","L5H1","L5H2","L5H3","L5H4","L5H5","L5H6","L5H7","L5H8","L5H9","L5H10","L5H11","L6H0","L6H1","L6H2","L6H3","L6H4","L6H5","L6H6","L6H7","L6H8","L6H9","L6H10","L6H11","L7H0","L7H1","L7H2","L7H3","L7H4","L7H5","L7H6","L7H7","L7H8","L7H9","L7H10","L7H11","L8H0","L8H1","L8H2","L8H3","L8H4","L8H5","L8H6","L8H7","L8H8","L8H9","L8H10","L8H11","L9H0","L9H1","L9H2","L9H3","L9H4","L9H5","L9H6","L9H7","L9H8","L9H9","L9H10","L9H11","L10H0","L10H1","L10H2","L10H3","L10H4","L10H5","L10H6","L10H7","L10H8","L10H9","L10H10","L10H11","L11H0","L11H1","L11H2","L11H3","L11H4","L11H5","L11H6","L11H7","L11H8","L11H9","L11H10","L11H11"],"legendgroup":"","marker":{"color":"#636efa","symbol":"circle"},"mode":"markers","name":"","orientation":"v","showlegend":false,"x":[0.0006398981786333025,0.005319283809512854,0.0011588380439206958,-5.923740172875114e-05,-0.0010669109178707004,0.005080018658190966,-0.0030812176410108805,-0.0020524668507277966,-0.0014406939735636115,0.003492556745186448,-0.0025678491219878197,-0.0009152094717137516,-0.0007601853576488793,0.0001696908293524757,0.00012250228610355407,-0.00034918184974230826,1.5405081285280176e-05,0.005008597858250141,-0.0002970260684378445,-0.0014443857362493873,-0.001099835499189794,0.00047436909517273307,5.108177356305532e-05,-0.003494771895930171,-0.0007234682561829686,0.0017475537024438381,-0.0001551919267512858,5.749216506956145e-05,-9.699495421955362e-05,-0.00042355581535957754,-0.000791196885984391,0.0002723913639783859,0.00010212998313363642,0.00042187771759927273,0.00015183570212684572,-0.0007432028651237488,0.11458060890436172,0.00021201284835115075,-0.000942025741096586,0.0004294627869967371,0.02004331909120083,0.002104991814121604,7.662265852559358e-05,-0.0015435622772201896,-0.0008476486545987427,-0.0005817683413624763,0.00011934742360608652,-1.9130493456032127e-05,-0.001126819639466703,0.0012377764796838164,-0.0012320708483457565,-0.0005953610525466502,-0.0007538085337728262,-0.0005843526450917125,0.004813097883015871,0.00018163899949286133,-0.0005363250384107232,0.0008580865105614066,-0.00029840212664566934,-1.2183103535789996e-05,-0.004240894690155983,0.0029526741709560156,0.0005220275488682091,0.0009532691328786314,0.0001625084987608716,0.3435087502002716,-0.0003038727736566216,0.00010364028275944293,-0.005300321150571108,0.024866117164492607,0.014383613131940365,-0.0023283488117158413,-0.0023879888467490673,-0.002172317821532488,-0.00047651707427576184,0.0004340272571425885,-0.004673780873417854,0.001858678413555026,-0.0026543056592345238,0.0014364985981956124,0.000303067296044901,0.1304299533367157,8.957769750850275e-05,0.0011777335312217474,0.0003194456803612411,0.020570717751979828,0.00031887509976513684,-0.002512404229491949,-0.0002621884341351688,-0.00024587716325186193,0.0005522335413843393,-0.00043134228326380253,0.0002570534124970436,0.008090788498520851,-0.003069269470870495,-0.0004241599526721984,0.0009768297895789146,0.00039294702582992613,0.0017541655106469989,0.022596266120672226,-4.443644138518721e-05,0.00014159921556711197,0.00958488043397665,-0.0003154182049911469,0.0015266804257407784,0.0011816268088296056,-0.01077402476221323,0.009365785866975784,0.006314304657280445,-0.001094431965611875,0.011661848053336143,0.0013478941982612014,-0.02918611839413643,0.0038340529426932335,-0.04409320279955864,-0.005031823180615902,0.004822427872568369,0.2766473889350891,-3.1280032999347895e-05,-0.0006619150517508388,0.09538879990577698,0.025069067254662514,0.014238725416362286,0.01475451048463583,9.914294059854001e-05,-8.977906691143289e-05,0.0508294515311718,-0.5050979852676392,0.00014717054727952927,-0.0016020277980715036,0.06883121281862259,0.0023271406535059214,0.0013427927624434233,0.009631095454096794,-0.07776330411434174,-0.007728920318186283,-0.0005721024353988469,-0.0029568022582679987,-0.00494667561724782,0.000458796217571944,-0.0006325145368464291,-0.006519974209368229,-0.32048866152763367,-0.002472163178026676],"xaxis":"x","y":[0.000949342327658087,0.016124119982123375,0.0018556242575868964,0.0034389242064207792,-0.009823138825595379,0.011059673503041267,-0.0040638200007379055,-0.0015785341383889318,-0.001208073808811605,0.003828749991953373,-0.004257441032677889,-0.0011428623693063855,-0.001076644053682685,-0.0003779782564379275,3.0206042538338806e-06,-0.000260510336374864,-0.00014149851631373167,0.0038328110240399837,-0.0004281202855054289,-0.0014291484840214252,-0.0009217541082762182,0.0006938663427717984,0.00043359093251638114,-0.0035708239302039146,-0.0004966544220224023,0.0008070718613453209,0.0005420306115411222,-0.0005315592279657722,-0.0007152119651436806,-0.0010380810126662254,-0.0009492752142250538,-8.538240945199504e-05,0.0002774928288999945,0.0021076432894915342,-0.0001976146304514259,-0.0016398524167016149,0.11626167595386505,0.00025017315056174994,-0.0014669395750388503,-0.0003969745012000203,0.01896214485168457,-0.00018882131553255022,0.011170395649969578,-0.0013304082676768303,-0.0007363561890088022,-0.00030310085276141763,-0.00014579448907170445,-0.0002225178322987631,-0.0016502231592312455,0.00029299859306775033,-0.0014352232683449984,0.030841777101159096,-0.007431894540786743,-0.0002820908557623625,0.006017110776156187,-0.011006947606801987,-0.0012651296565309167,0.001490030437707901,-0.00018016225658357143,0.0029450219590216875,-0.004210856277495623,0.0029608632903546095,0.002045620232820511,0.0013410139363259077,-0.0012189144035801291,0.34349939227104187,0.0005627049831673503,-0.0001257913827430457,-0.005152345169335604,0.016241183504462242,0.017089638859033585,-0.004173669498413801,0.039775215089321136,0.015227536670863628,-0.001023011514917016,0.0008081123232841492,-0.004935599863529205,-0.002123551908880472,-0.014273260720074177,0.0013749118661507964,0.0014838549541309476,0.1302708089351654,-0.00033538774005137384,0.0012914760736748576,0.0003724069392774254,0.0195147804915905,0.00022308839834295213,0.12424899637699127,-0.0004041232750751078,-0.007652163505554199,0.0013012762647122145,-0.0011244030902162194,-0.007449581753462553,0.19224178791046143,-0.0032753082923591137,-0.000501957256346941,-0.001007740618661046,3.1212908652378246e-05,-0.0008586570620536804,0.012360110878944397,-0.00040429108776152134,-0.004328995477408171,0.31855517625808716,0.002329523442313075,0.002118081087246537,0.0001410957775078714,0.27793484926223755,0.005738543812185526,0.0058897752314805984,-0.000968372100032866,0.009125681594014168,0.020675498992204666,-0.03700448200106621,0.014263628050684929,-0.04828385263681412,0.05834108963608742,0.0006516450084745884,0.2636081576347351,0.0004918886115774512,-0.0026110773906111717,0.08374276012182236,0.020676370710134506,-0.0037437702994793653,0.01085111778229475,-0.0010960765648633242,0.0004738992138300091,0.04817944020032883,-0.47990989685058594,0.00018412260396871716,0.011861777864396572,0.06088537722826004,0.0008459034143015742,0.005328110884875059,-0.011493667028844357,-0.11350765824317932,0.006330112461000681,0.0003166264505125582,-0.0011600126745179296,-0.022669030353426933,0.004070901777595282,0.007316238712519407,-0.008345123380422592,-0.27817434072494507,0.003635196480900049],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"Attention Patch"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"Output Patch"}},"legend":{"tracegroupgap":0},"title":{"text":"Scatter plot of output patching vs attention patching"}},                        {"responsive": true}                    ).then(function(){
+<div>                            <div id="fc3a3636-e1a7-4d2b-b7d1-f71b5cc7bbb7" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("fc3a3636-e1a7-4d2b-b7d1-f71b5cc7bbb7")) {                    Plotly.newPlot(                        "fc3a3636-e1a7-4d2b-b7d1-f71b5cc7bbb7",                        [{"hovertemplate":"\u003cb\u003e%{hovertext}\u003c\u002fb\u003e\u003cbr\u003e\u003cbr\u003eAttention Patch=%{x}\u003cbr\u003eOutput Patch=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","hovertext":["L0H0","L0H1","L0H2","L0H3","L0H4","L0H5","L0H6","L0H7","L0H8","L0H9","L0H10","L0H11","L1H0","L1H1","L1H2","L1H3","L1H4","L1H5","L1H6","L1H7","L1H8","L1H9","L1H10","L1H11","L2H0","L2H1","L2H2","L2H3","L2H4","L2H5","L2H6","L2H7","L2H8","L2H9","L2H10","L2H11","L3H0","L3H1","L3H2","L3H3","L3H4","L3H5","L3H6","L3H7","L3H8","L3H9","L3H10","L3H11","L4H0","L4H1","L4H2","L4H3","L4H4","L4H5","L4H6","L4H7","L4H8","L4H9","L4H10","L4H11","L5H0","L5H1","L5H2","L5H3","L5H4","L5H5","L5H6","L5H7","L5H8","L5H9","L5H10","L5H11","L6H0","L6H1","L6H2","L6H3","L6H4","L6H5","L6H6","L6H7","L6H8","L6H9","L6H10","L6H11","L7H0","L7H1","L7H2","L7H3","L7H4","L7H5","L7H6","L7H7","L7H8","L7H9","L7H10","L7H11","L8H0","L8H1","L8H2","L8H3","L8H4","L8H5","L8H6","L8H7","L8H8","L8H9","L8H10","L8H11","L9H0","L9H1","L9H2","L9H3","L9H4","L9H5","L9H6","L9H7","L9H8","L9H9","L9H10","L9H11","L10H0","L10H1","L10H2","L10H3","L10H4","L10H5","L10H6","L10H7","L10H8","L10H9","L10H10","L10H11","L11H0","L11H1","L11H2","L11H3","L11H4","L11H5","L11H6","L11H7","L11H8","L11H9","L11H10","L11H11"],"legendgroup":"","marker":{"color":"#636efa","symbol":"circle"},"mode":"markers","name":"","orientation":"v","showlegend":false,"x":[0.0006398981786333025,0.005319283809512854,0.0011588380439206958,-5.923740172875114e-05,-0.0010669109178707004,0.005080018658190966,-0.0030812176410108805,-0.0020524668507277966,-0.0014406939735636115,0.003492556745186448,-0.0025678491219878197,-0.0009152094717137516,-0.0007601853576488793,0.0001696908293524757,0.00012250228610355407,-0.00034918184974230826,1.5405081285280176e-05,0.005008597858250141,-0.0002970260684378445,-0.0014443857362493873,-0.001099835499189794,0.00047436909517273307,5.108177356305532e-05,-0.003494771895930171,-0.0007234682561829686,0.0017475537024438381,-0.0001551919267512858,5.749216506956145e-05,-9.699495421955362e-05,-0.00042355581535957754,-0.000791196885984391,0.0002723913639783859,0.00010212998313363642,0.00042187771759927273,0.00015183570212684572,-0.0007432028651237488,0.11458060890436172,0.00021201284835115075,-0.000942025741096586,0.0004294627869967371,0.02004331909120083,0.002104991814121604,7.662265852559358e-05,-0.0015435622772201896,-0.0008476486545987427,-0.0005817683413624763,0.00011934742360608652,-1.9130493456032127e-05,-0.001126819639466703,0.0012377764796838164,-0.0012320708483457565,-0.0005953610525466502,-0.0007538085337728262,-0.0005843526450917125,0.004813097883015871,0.00018163899949286133,-0.0005363250384107232,0.0008580865105614066,-0.00029840212664566934,-1.2183103535789996e-05,-0.004240894690155983,0.0029526741709560156,0.0005220275488682091,0.0009532691328786314,0.0001625084987608716,0.3435087502002716,-0.0003038727736566216,0.00010364028275944293,-0.005300321150571108,0.024866117164492607,0.014383613131940365,-0.0023283488117158413,-0.0023879888467490673,-0.002172317821532488,-0.00047651707427576184,0.0004340272571425885,-0.004673780873417854,0.001858678413555026,-0.0026543056592345238,0.0014364985981956124,0.000303067296044901,0.1304299533367157,8.957769750850275e-05,0.0011777335312217474,0.0003194456803612411,0.020570717751979828,0.00031887509976513684,-0.002512404229491949,-0.0002621884341351688,-0.00024587716325186193,0.0005522335413843393,-0.00043134228326380253,0.0002570534124970436,0.008090788498520851,-0.003069269470870495,-0.0004241599526721984,0.0009768297895789146,0.00039294702582992613,0.0017541655106469989,0.022596266120672226,-4.443644138518721e-05,0.00014159921556711197,0.00958488043397665,-0.0003154182049911469,0.0015266804257407784,0.0011816268088296056,-0.01077402476221323,0.009365785866975784,0.006314304657280445,-0.001094431965611875,0.011661848053336143,0.0013478941982612014,-0.02918611839413643,0.0038340529426932335,-0.04409320279955864,-0.005031823180615902,0.004822427872568369,0.2766473889350891,-3.1280032999347895e-05,-0.0006619150517508388,0.09538879990577698,0.025069067254662514,0.014238725416362286,0.01475451048463583,9.914294059854001e-05,-8.977906691143289e-05,0.0508294515311718,-0.5050979852676392,0.00014717054727952927,-0.0016020277980715036,0.06883121281862259,0.0023271406535059214,0.0013427927624434233,0.009631095454096794,-0.07776330411434174,-0.007728920318186283,-0.0005721024353988469,-0.0029568022582679987,-0.00494667561724782,0.000458796217571944,-0.0006325145368464291,-0.006519974209368229,-0.32048866152763367,-0.002472163178026676],"xaxis":"x","y":[0.000949342327658087,0.016124119982123375,0.0018556242575868964,0.0034389242064207792,-0.009823138825595379,0.011059673503041267,-0.0040638200007379055,-0.0015785341383889318,-0.001208073808811605,0.003828749991953373,-0.004257441032677889,-0.0011428623693063855,-0.001076644053682685,-0.0003779782564379275,3.0206042538338806e-06,-0.000260510336374864,-0.00014149851631373167,0.0038328110240399837,-0.0004281202855054289,-0.0014291484840214252,-0.0009217541082762182,0.0006938663427717984,0.00043359093251638114,-0.0035708239302039146,-0.0004966544220224023,0.0008070718613453209,0.0005420306115411222,-0.0005315592279657722,-0.0007152119651436806,-0.0010380810126662254,-0.0009492752142250538,-8.538240945199504e-05,0.0002774928288999945,0.0021076432894915342,-0.0001976146304514259,-0.0016398524167016149,0.11626167595386505,0.00025017315056174994,-0.0014669395750388503,-0.0003969745012000203,0.01896214485168457,-0.00018882131553255022,0.011170395649969578,-0.0013304082676768303,-0.0007363561890088022,-0.00030310085276141763,-0.00014579448907170445,-0.0002225178322987631,-0.0016502231592312455,0.00029299859306775033,-0.0014352232683449984,0.030841777101159096,-0.007431894540786743,-0.0002820908557623625,0.006017110776156187,-0.011006947606801987,-0.0012651296565309167,0.001490030437707901,-0.00018016225658357143,0.0029450219590216875,-0.004210856277495623,0.0029608632903546095,0.002045620232820511,0.0013410139363259077,-0.0012189144035801291,0.34349939227104187,0.0005627049831673503,-0.0001257913827430457,-0.005152345169335604,0.016241183504462242,0.017089638859033585,-0.004173669498413801,0.039775215089321136,0.015227536670863628,-0.001023011514917016,0.0008081123232841492,-0.004935599863529205,-0.002123551908880472,-0.014273260720074177,0.0013749118661507964,0.0014838549541309476,0.1302708089351654,-0.00033538774005137384,0.0012914760736748576,0.0003724069392774254,0.0195147804915905,0.00022308839834295213,0.12424899637699127,-0.0004041232750751078,-0.007652163505554199,0.0013012762647122145,-0.0011244030902162194,-0.007449581753462553,0.19224178791046143,-0.0032753082923591137,-0.000501957256346941,-0.001007740618661046,3.1212908652378246e-05,-0.0008586570620536804,0.012360110878944397,-0.00040429108776152134,-0.004328995477408171,0.31855517625808716,0.002329523442313075,0.002118081087246537,0.0001410957775078714,0.27793484926223755,0.005738543812185526,0.0058897752314805984,-0.000968372100032866,0.009125681594014168,0.020675498992204666,-0.03700448200106621,0.014263628050684929,-0.04828385263681412,0.05834108963608742,0.0006516450084745884,0.2636081576347351,0.0004918886115774512,-0.0026110773906111717,0.08374276012182236,0.020676370710134506,-0.0037437702994793653,0.01085111778229475,-0.0010960765648633242,0.0004738992138300091,0.04817944020032883,-0.47990989685058594,0.00018412260396871716,0.011861777864396572,0.06088537722826004,0.0008459034143015742,0.005328110884875059,-0.011493667028844357,-0.11350765824317932,0.006330112461000681,0.0003166264505125582,-0.0011600126745179296,-0.022669030353426933,0.004070901777595282,0.007316238712519407,-0.008345123380422592,-0.27817434072494507,0.003635196480900049],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"Attention Patch"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"Output Patch"}},"legend":{"tracegroupgap":0},"title":{"text":"Scatter plot of output patching vs attention patching"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('f93bc8e7-172e-489a-86d5-ed9f1696b43b');
+var gd = document.getElementById('fc3a3636-e1a7-4d2b-b7d1-f71b5cc7bbb7');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -1853,27 +1853,27 @@ <h3>Visualizing Attention Patterns<a class="headerlink" href="#Visualizing-Atten
 </pre></div>
 </div>
 <div class="output_area rendered_html docutils container">
-<div style='max-width: 700px;'><h2>Top Early Heads</h2><br/><div id="circuits-vis-8e46332b-2641" style="margin: 15px 0;"/>
+<div style='max-width: 700px;'><h2>Top Early Heads</h2><br/><div id="circuits-vis-49c7fd51-4c0a" style="margin: 15px 0;"/>
     <script crossorigin type="module">
     import { render, AttentionHeads } from "https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js";
     render(
-      "circuits-vis-8e46332b-2641",
+      "circuits-vis-49c7fd51-4c0a",
       AttentionHeads,
       {"attention": [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9964697360992432, 0.0035302219912409782, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9966080188751221, 0.0017803364899009466, 0.0016116512706503272, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9577597975730896, 0.002041348023340106, 0.00576710794121027, 0.03443171828985214, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9969283938407898, 0.0006081241299398243, 0.0016115037724375725, 0.0005201429594308138, 0.0003317868977319449, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9679325819015503, 0.0013422606280073524, 0.011834591627120972, 0.004275884013622999, 0.005249680485576391, 0.009365071542561054, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9869164228439331, 0.0013478520559147, 0.008249778300523758, 0.00044454942690208554, 0.0012886020122095942, 0.0008496184600517154, 0.0009031315566971898, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9841058850288391, 0.005145575385540724, 0.0032505656126886606, 0.00023865938419476151, 0.0014756217133253813, 0.000972211651969701, 0.00134052406065166, 0.003471038304269314, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9905135631561279, 0.0026424615643918514, 0.0020855581387877464, 4.1533587591402465e-07, 1.741811320243869e-05, 0.0003096052969340235, 0.00018154106510337442, 0.0010755121475085616, 0.0031737794633954763, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9528716206550598, 0.00732765905559063, 0.008451228961348534, 0.00028046435909345746, 0.002839941531419754, 0.002773231128230691, 0.001109760720282793, 0.0011476994259282947, 0.0057571399956941605, 0.017441321164369583, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04626865312457085, 0.00023600729764439166, 0.00012578870519064367, 0.5745652914047241, 0.3546755015850067, 0.0192879606038332, 9.036171832121909e-05, 0.00044691169750876725, 0.004133324604481459, 3.63319622920244e-06, 0.00016656972002238035, 0.0, 0.0, 0.0, 0.0], [0.887678861618042, 0.005508160684257746, 0.008590048179030418, 0.022789321839809418, 0.034917183220386505, 0.001486042165197432, 7.143727270886302e-05, 0.0006066859932616353, 0.010601978749036789, 0.0001475940807722509, 0.004989828448742628, 0.022612828761339188, 0.0, 0.0, 0.0], [0.9325922131538391, 0.009143135510385036, 0.0023391752038151026, 0.0008593837847001851, 0.0012484431499615312, 0.00037233170587569475, 0.0001396290463162586, 5.667153527610935e-05, 0.029758313670754433, 0.002666445914655924, 0.00340129598043859, 0.0010437102755531669, 0.016379283741116524, 0.0, 0.0], [0.9817627668380737, 0.001917588640935719, 0.00040775517118163407, 1.3564374512498034e-07, 1.264090315089561e-05, 2.9349219403229654e-05, 3.720399035955779e-05, 1.9605669876909815e-05, 0.00034280188265256584, 0.0020146335009485483, 0.007377031724900007, 4.751105734612793e-06, 0.0004324739275034517, 0.005641316995024681, 0.0], [0.908775269985199, 0.004100896418094635, 0.0067780399695038795, 1.580042953719385e-05, 0.00011193934915354475, 8.643363980809227e-05, 3.187262109349831e-06, 0.001585155026987195, 0.003799399361014366, 0.0067048450000584126, 0.0139771094545722, 0.0001160897736554034, 0.004422870930284262, 0.021179812029004097, 0.028343213722109795]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9990116357803345, 0.0009883929742500186, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9985806941986084, 0.0004404807696118951, 0.0009789254982024431, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9970335960388184, 0.0004637633392121643, 0.0009368518949486315, 0.0015657964395359159, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9892035126686096, 8.868379518389702e-05, 0.000574222591239959, 0.009837915189564228, 0.0002956092939712107, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9954046010971069, 0.00010050175478681922, 0.0013415871653705835, 3.886737977154553e-05, 0.0001206161396112293, 0.00299392850138247, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9989855885505676, 0.00010351995297241956, 0.0004172757326159626, 7.227508376672631e-06, 6.046874841558747e-05, 0.0002549678029026836, 0.0001708982017589733, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9980852603912354, 0.00016822919133119285, 0.00010910424316534773, 4.1012435758602805e-06, 5.103566945763305e-05, 0.0001056204637279734, 4.150096719968133e-05, 0.0014351928839460015, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9972643852233887, 7.076081237755716e-05, 6.327207665890455e-05, 2.615450966914068e-08, 5.106949174660258e-06, 3.7383753806352615e-05, 3.455430851317942e-05, 0.000461774121504277, 0.0020626918412745, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9859043955802917, 0.0015692580491304398, 0.001985961804166436, 5.8834184528677724e-06, 0.0003162050561513752, 0.00014976764214225113, 2.4109986043185927e-05, 0.00012667303963098675, 0.0012655562022700906, 0.00865224003791809, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3212882876396179, 0.0026113800704479218, 0.0017525857547298074, 0.48284193873405457, 0.1752953678369522, 0.012053580023348331, 5.7314944569952786e-05, 0.00047417485620826483, 0.0011855909833684564, 4.6230838051997125e-05, 0.0023935972712934017, 0.0, 0.0, 0.0, 0.0], [0.9928818941116333, 0.0011452743783593178, 0.0015605579828843474, 5.570496796281077e-05, 0.0020913302432745695, 0.0002502372954040766, 9.751672678248724e-07, 1.4541505152010359e-05, 0.0006855707615613937, 3.3003976568579674e-05, 0.0003418456471990794, 0.0009391428320668638, 0.0, 0.0, 0.0], [0.9924144148826599, 0.0006524075288325548, 0.00016266616876237094, 7.948347956698854e-06, 0.00015110620006453246, 6.227314588613808e-05, 7.466151714652369e-07, 4.000507090040628e-07, 0.00244068237952888, 3.030487641808577e-05, 4.814808562514372e-05, 9.094910637941211e-05, 0.003938092850148678, 0.0, 0.0], [0.9971888661384583, 0.00062146270647645, 3.946273500332609e-05, 1.9362538239420246e-08, 1.800630002435355e-06, 1.9080694983131252e-05, 9.158955435850658e-06, 7.050003887343337e-07, 6.766093429178e-05, 0.0008072670316323638, 0.00013612494512926787, 2.204007842010469e-06, 2.116413634212222e-05, 0.0010850975522771478, 0.0], [0.9784426689147949, 0.0038412772119045258, 0.001794080133549869, 9.544757631374523e-06, 0.0001613280619494617, 0.00021571738761849701, 3.438441638081713e-07, 0.0001911988656502217, 0.0018583982018753886, 0.002262116875499487, 0.0028807998169213533, 0.00015137717127799988, 3.800494960159995e-05, 0.004419004078954458, 0.0037341148126870394]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9793283343315125, 0.020671749487519264, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.906032145023346, 0.0005559856654144824, 0.09341184794902802, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9916713237762451, 0.0009248568094335496, 0.0019154187757521868, 0.005488428752869368, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9500631093978882, 0.0015093885594978929, 0.00022879372409079224, 0.0011742664501070976, 0.04702451452612877, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9742831587791443, 0.001334858825430274, 0.00016252745990641415, 0.001139267929829657, 1.2687537491729017e-06, 0.023079030215740204, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9734269380569458, 0.0036638062447309494, 0.0007961394148878753, 0.0020336511079221964, 1.7259832020499744e-05, 0.013762583956122398, 0.006299513392150402, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9740443229675293, 0.0018358685774728656, 0.00031735500670038164, 0.0013147101271897554, 4.377634832053445e-05, 0.00045178644359111786, 0.00035931551246903837, 0.021632865071296692, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8537226319313049, 0.0009385455632582307, 1.8928734789369628e-05, 0.0007116434862837195, 4.0574541344540194e-05, 2.706850500544533e-05, 0.0005688024684786797, 0.01303892396390438, 0.1309327930212021, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9718613028526306, 0.0030212444253265858, 0.0018890955252572894, 0.0058778319507837296, 0.0006584360380657017, 0.002494583837687969, 0.001156346406787634, 0.008928273804485798, 0.0013289419002830982, 0.0027839455287903547, 0.0, 0.0, 0.0, 0.0, 0.0], [0.21994593739509583, 0.0028093564324080944, 0.6635686159133911, 0.007326209452003241, 0.001153560820966959, 0.0018521154997870326, 0.0013647314626723528, 0.008370185270905495, 0.0001247532054549083, 0.002670685760676861, 0.09081388264894485, 0.0, 0.0, 0.0, 0.0], [0.9398402571678162, 0.0037398827262222767, 0.0006579139735549688, 0.0027551481034606695, 1.3964868230686989e-05, 0.002081213751807809, 0.007874594070017338, 0.008091994561254978, 0.002785064745694399, 0.003697305917739868, 0.00012857612455263734, 0.028334153816103935, 0.0, 0.0, 0.0], [0.9256278872489929, 0.003680802881717682, 0.0006159098702482879, 0.004310118965804577, 7.914558227639645e-05, 0.0015225791139528155, 0.0014093437930569053, 0.041066594421863556, 0.0011927677551284432, 0.002704742830246687, 8.953329233918339e-05, 0.00031252068583853543, 0.017388159409165382, 0.0, 0.0], [0.7785975933074951, 0.005006679333746433, 8.588898344896734e-05, 0.0016439110040664673, 3.517184086376801e-05, 0.0007985705742612481, 0.004088219255208969, 0.03206512704491615, 0.02768608182668686, 0.003920219838619232, 2.586934351711534e-05, 0.0010157276410609484, 0.009296733886003494, 0.13573415577411652, 0.0], [0.954503059387207, 0.006336927879601717, 0.00032585320877842605, 0.0035448195412755013, 2.0907922589685768e-05, 0.007449527736753225, 0.009912977926433086, 0.005456207785755396, 0.0005529882037080824, 0.0011178080458194017, 6.70458102831617e-05, 0.003978283144533634, 0.0024476239923387766, 6.584440416190773e-05, 0.00422010850161314]]], "attentionHeadNames": ["L5H5", "L6H9", "L3H0"], "tokens": ["<|endoftext|>", "When", " John", " and", " Mary", " went", " to", " the", " shops", ",", " John", " gave", " the", " bag", " to"]}
     )
-    </script></div><div style='max-width: 700px;'><h2>Top Middle Heads</h2><br/><div id="circuits-vis-db6b15e4-7a16" style="margin: 15px 0;"/>
+    </script></div><div style='max-width: 700px;'><h2>Top Middle Heads</h2><br/><div id="circuits-vis-c23f4f46-0e8b" style="margin: 15px 0;"/>
     <script crossorigin type="module">
     import { render, AttentionHeads } from "https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js";
     render(
-      "circuits-vis-db6b15e4-7a16",
+      "circuits-vis-c23f4f46-0e8b",
       AttentionHeads,
       {"attention": [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9883801937103271, 0.011619755066931248, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9756757020950317, 0.006998097989708185, 0.017326191067695618, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.06910058110952377, 0.005814341828227043, 0.9046230316162109, 0.020462043583393097, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8063386082649231, 0.003319838549941778, 0.0537770576775074, 0.11000825464725494, 0.02655618265271187, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9564918875694275, 0.004251283593475819, 0.011477907188236713, 0.011219605803489685, 0.005006618797779083, 0.011552776210010052, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9077332615852356, 0.0036374430637806654, 0.040214236825704575, 0.012946903705596924, 0.019611867144703865, 0.011057589203119278, 0.004798637703061104, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9552405476570129, 0.009812522679567337, 0.01168749388307333, 0.0090243611484766, 0.0034366166219115257, 0.004399922210723162, 0.0033402289263904095, 0.003058257745578885, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9294586777687073, 0.009998402558267117, 0.004449557047337294, 0.006221562158316374, 0.0014876301866024733, 0.018290331587195396, 0.00712206494063139, 0.0036232024431228638, 0.0193486325442791, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6007111668586731, 0.014475381001830101, 0.09337030351161957, 0.01660209521651268, 0.05098922923207283, 0.10098620504140854, 0.051671840250492096, 0.02743108756840229, 0.02603602036833763, 0.017726697027683258, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6264241933822632, 0.012645086273550987, 0.016070131212472916, 0.024194441735744476, 0.029195599257946014, 0.11857090145349503, 0.06370382010936737, 0.023378603160381317, 0.001658876077271998, 0.06528978794813156, 0.018868643790483475, 0.0, 0.0, 0.0, 0.0], [0.2991042137145996, 0.0021427334286272526, 0.024437544867396355, 0.005155442748218775, 0.010526876896619797, 0.003321276046335697, 0.009254480712115765, 0.005843525752425194, 0.0021181809715926647, 0.017681695520877838, 0.6048213839530945, 0.015592672862112522, 0.0, 0.0, 0.0], [0.8641641139984131, 0.004937537480145693, 0.019023817032575607, 0.001670254161581397, 0.00609729764983058, 0.0005472623743116856, 0.003564003622159362, 0.00030002548010088503, 0.007773987948894501, 0.007786957547068596, 0.06421297788619995, 0.005789309274405241, 0.014132493175566196, 0.0, 0.0], [0.9178580045700073, 0.005138528533279896, 0.003403181442990899, 0.0021888096816837788, 0.002614964498206973, 0.0028868599329143763, 0.0028416835702955723, 0.0013745100004598498, 0.006431094836443663, 0.012306848540902138, 0.0090817641466856, 0.01310056634247303, 0.008110282011330128, 0.012662900611758232, 0.0], [0.019487837329506874, 5.700236215488985e-05, 0.026276404038071632, 0.0009103945340029895, 0.0056328438222408295, 7.981786620803177e-05, 5.7736750022741035e-05, 0.000307683803839609, 0.00014904775889590383, 0.0002852973702829331, 0.9430450201034546, 0.0007767666247673333, 0.0018659740453585982, 0.0006072799442335963, 0.0004609254829119891]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9562382698059082, 0.043761663138866425, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9643059372901917, 0.00742809334769845, 0.028266001492738724, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6456211805343628, 0.029121818020939827, 0.20926059782505035, 0.11599643528461456, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7634043097496033, 0.01714019477367401, 0.026512376964092255, 0.08158667385578156, 0.11135648936033249, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8267454504966736, 0.029910113662481308, 0.015236063860356808, 0.034572526812553406, 0.027030833065509796, 0.06650502979755402, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7882238626480103, 0.02833343856036663, 0.04124196991324425, 0.024983419105410576, 0.05788382515311241, 0.020290205255150795, 0.03904324769973755, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7217290997505188, 0.027771806344389915, 0.02081979066133499, 0.024504197761416435, 0.01813969761133194, 0.014079142361879349, 0.04442400485277176, 0.12853237986564636, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7728042602539062, 0.04133640229701996, 0.006504536606371403, 0.007949033752083778, 0.006013217847794294, 0.01562037505209446, 0.03980163484811783, 0.05545476824045181, 0.054515715688467026, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.47073426842689514, 0.03082239255309105, 0.07406120002269745, 0.055252522230148315, 0.14865727722644806, 0.062336407601833344, 0.034053124487400055, 0.026126855984330177, 0.042438995093107224, 0.05551688000559807, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6962025165557861, 0.0052972170524299145, 0.004229621961712837, 0.05190793052315712, 0.007751153316348791, 0.03517890349030495, 0.03057355433702469, 0.021498825401067734, 0.01566012017428875, 0.051384858787059784, 0.08031534403562546, 0.0, 0.0, 0.0, 0.0], [0.37817656993865967, 0.006224599666893482, 0.023588571697473526, 0.044115372002124786, 0.059451501816511154, 0.00953902117908001, 0.012997719459235668, 0.014349035918712616, 0.0203937329351902, 0.01785479485988617, 0.33459627628326416, 0.07871285080909729, 0.0, 0.0, 0.0], [0.4760020971298218, 0.011026055552065372, 0.018324486911296844, 0.04565680772066116, 0.026791490614414215, 0.011260150000452995, 0.018724240362644196, 0.018448904156684875, 0.04068983346223831, 0.023096060380339622, 0.12447667866945267, 0.08994489908218384, 0.0955582857131958, 0.0, 0.0], [0.7641531229019165, 0.00824385043233633, 0.0032178766559809446, 0.004948926158249378, 0.004036621656268835, 0.00750678451731801, 0.011691190302371979, 0.013843986205756664, 0.01143654901534319, 0.0222361721098423, 0.013085336424410343, 0.03330926224589348, 0.036730363965034485, 0.06555990874767303, 0.0], [0.1303366869688034, 0.0022374859545379877, 0.0373632088303566, 0.06385007500648499, 0.11956273019313812, 0.011286774650216103, 0.006565204821527004, 0.00851503573358059, 0.005633368622511625, 0.007633764296770096, 0.5373668074607849, 0.0301155224442482, 0.020915960893034935, 0.005129658151417971, 0.013487689197063446]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.972596287727356, 0.027403652667999268, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9560405015945435, 0.037302762269973755, 0.006656836252659559, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5009381771087646, 0.0898313894867897, 0.34842997789382935, 0.06080043688416481, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7674180269241333, 0.17525732517242432, 0.021164551377296448, 0.02390032634139061, 0.012259737588465214, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7222559452056885, 0.18388347327709198, 0.023735947906970978, 0.007100591901689768, 0.007225400768220425, 0.05579861253499985, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6940793395042419, 0.13069462776184082, 0.028726812452077866, 0.009216434322297573, 0.011527394875884056, 0.08933128416538239, 0.036424122750759125, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6924421191215515, 0.12459339946508408, 0.01531891617923975, 0.007925059646368027, 0.006713555194437504, 0.0796525627374649, 0.04910353198647499, 0.02425084263086319, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6969695091247559, 0.10134164988994598, 0.011634294874966145, 0.007826745510101318, 0.0032780379988253117, 0.042647555470466614, 0.05842007324099541, 0.042913682758808136, 0.03496843948960304, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6869087815284729, 0.023936893790960312, 0.10650163143873215, 0.02047020196914673, 0.02552187815308571, 0.04722552374005318, 0.022190820425748825, 0.007246002089232206, 0.019754212349653244, 0.04024401307106018, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5810369849205017, 0.011352411471307278, 0.0033976146951317787, 0.0365644209086895, 0.020549381151795387, 0.051692016422748566, 0.024651870131492615, 0.015468521043658257, 0.049509111791849136, 0.15767459571361542, 0.04810298979282379, 0.0, 0.0, 0.0, 0.0], [0.43488532304763794, 0.006897220853716135, 0.012041947804391384, 0.003814741736277938, 0.010545267723500729, 0.0076634688302874565, 0.007842864841222763, 0.006417531985789537, 0.044214341789484024, 0.10391034185886383, 0.3332052230834961, 0.028561806306242943, 0.0, 0.0, 0.0], [0.4747830927371979, 0.024993136525154114, 0.009554888121783733, 0.0037503365892916918, 0.008160687983036041, 0.02339756116271019, 0.010678932070732117, 0.0014935131184756756, 0.030863573774695396, 0.19498929381370544, 0.15084564685821533, 0.054150503128767014, 0.012338940985500813, 0.0, 0.0], [0.7144870758056641, 0.009456266649067402, 0.0035061524249613285, 0.0029858576599508524, 0.001229096669703722, 0.01199608389288187, 0.008696629665791988, 0.007586767431348562, 0.009631272405385971, 0.06210443004965782, 0.014414231292903423, 0.07928328961133957, 0.04383841156959534, 0.030784491449594498, 0.0], [0.1013382077217102, 0.0038837045431137085, 0.01792380027472973, 0.0038827748503535986, 0.009177407249808311, 0.01316938828676939, 0.0034797503612935543, 0.0019711176864802837, 0.004929723683744669, 0.1296602189540863, 0.6630275249481201, 0.0330243818461895, 0.0089839156717062, 0.0017172213410958648, 0.003830986563116312]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.97651207447052, 0.023487908765673637, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9743125438690186, 0.009410635568201542, 0.016276877373456955, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5441411137580872, 0.019679147750139236, 0.33595189452171326, 0.10022785514593124, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8594159483909607, 0.021821973845362663, 0.025710705667734146, 0.03842515870928764, 0.05462610721588135, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7619735598564148, 0.07210072875022888, 0.037441689521074295, 0.04506158083677292, 0.031744565814733505, 0.05167779326438904, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8025401830673218, 0.026160430163145065, 0.033131979405879974, 0.02643769420683384, 0.03139319270849228, 0.04932845011353493, 0.031008044257760048, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7639790773391724, 0.021559985354542732, 0.016495496034622192, 0.025408243760466576, 0.027263421565294266, 0.06810054928064346, 0.047422830015420914, 0.029770376160740852, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7543362379074097, 0.01597309671342373, 0.008247380144894123, 0.00967292208224535, 0.013097813352942467, 0.03411843255162239, 0.03947027027606964, 0.032298993319272995, 0.09278487414121628, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5856387615203857, 0.0213167741894722, 0.026237597689032555, 0.020694918930530548, 0.02824069745838642, 0.11104536801576614, 0.047170888632535934, 0.022986045107245445, 0.04925474151968956, 0.08741410821676254, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6466761231422424, 0.015195527113974094, 0.00594041682779789, 0.02527434751391411, 0.010863534174859524, 0.08686516433954239, 0.032730914652347565, 0.021324681118130684, 0.04200880602002144, 0.09539096057415009, 0.017729468643665314, 0.0, 0.0, 0.0, 0.0], [0.42771559953689575, 0.014421668834984303, 0.024129344150424004, 0.026094987988471985, 0.01488838903605938, 0.010527816601097584, 0.012212568894028664, 0.015342887490987778, 0.017334001138806343, 0.04439026489853859, 0.28982794284820557, 0.10311443358659744, 0.0, 0.0, 0.0], [0.4405166506767273, 0.018903780728578568, 0.010929570533335209, 0.015466002747416496, 0.013489533215761185, 0.020115477964282036, 0.019655633717775345, 0.005925851874053478, 0.022772811353206635, 0.06206439808011055, 0.16126078367233276, 0.17319339513778687, 0.03570614382624626, 0.0, 0.0], [0.7304301261901855, 0.008421944454312325, 0.0022352896630764008, 0.004027598071843386, 0.0036414540372788906, 0.009633221663534641, 0.0213489830493927, 0.021966980770230293, 0.02610298991203308, 0.05093422904610634, 0.0083627263084054, 0.023242846131324768, 0.038515783846378326, 0.051135752350091934, 0.0], [0.16577893495559692, 0.004480816423892975, 0.0077864062041044235, 0.007284588646143675, 0.01337976660579443, 0.019251490011811256, 0.010158197022974491, 0.01172470673918724, 0.011870641261339188, 0.034697193652391434, 0.1448296457529068, 0.4143996834754944, 0.08938297629356384, 0.019558662548661232, 0.04541636258363724]]], "attentionHeadNames": ["L8H6", "L8H10", "L7H9", "L7H3"], "tokens": ["<|endoftext|>", "When", " John", " and", " Mary", " went", " to", " the", " shops", ",", " John", " gave", " the", " bag", " to"]}
     )
-    </script></div><div style='max-width: 700px;'><h2>Top Late Heads</h2><br/><div id="circuits-vis-7ba2561c-b325" style="margin: 15px 0;"/>
+    </script></div><div style='max-width: 700px;'><h2>Top Late Heads</h2><br/><div id="circuits-vis-9c2de0cf-406e" style="margin: 15px 0;"/>
     <script crossorigin type="module">
     import { render, AttentionHeads } from "https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js";
     render(
-      "circuits-vis-7ba2561c-b325",
+      "circuits-vis-9c2de0cf-406e",
       AttentionHeads,
       {"attention": [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9628916382789612, 0.03710832819342613, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9778335094451904, 0.0034864130429923534, 0.018680110573768616, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8737697601318359, 0.005964328069239855, 0.08126737922430038, 0.03899851813912392, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8745138645172119, 0.017276111990213394, 0.018874678760766983, 0.054268334060907364, 0.035066962242126465, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9704693555831909, 0.0035177830141037703, 0.0003239834331907332, 0.004373463336378336, 0.0003953034174628556, 0.02092011272907257, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7694658041000366, 0.006685878615826368, 0.011966485530138016, 0.040966276079416275, 0.03344443440437317, 0.04401002824306488, 0.0934610366821289, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8177722692489624, 0.009112595580518246, 0.0042336746118962765, 0.03900325670838356, 0.012653318233788013, 0.007337852846831083, 0.07410021126270294, 0.0357869453728199, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8309933543205261, 0.018796611577272415, 0.00330327614210546, 0.01983213610947132, 0.004009497817605734, 0.012078307569026947, 0.03246007487177849, 0.010322270914912224, 0.06820447742938995, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.31403300166130066, 0.01522789802402258, 0.19789263606071472, 0.059379592537879944, 0.3236042559146881, 0.009037697687745094, 0.019917670637369156, 0.012475269846618176, 0.013939663767814636, 0.03449229896068573, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7319568991661072, 0.03087025322020054, 0.005064091179519892, 0.08703826367855072, 0.004982746671885252, 0.044725559651851654, 0.02350614219903946, 0.0063977413810789585, 0.0034696452785283327, 0.048653144389390945, 0.013335512951016426, 0.0, 0.0, 0.0, 0.0], [0.13710583746433258, 0.012017901986837387, 0.14592327177524567, 0.015683427453041077, 0.44125232100486755, 0.004342416767030954, 0.011117888614535332, 0.007466601673513651, 0.0012341630645096302, 0.01946023479104042, 0.16842849552631378, 0.035967420786619186, 0.0, 0.0, 0.0], [0.4052484929561615, 0.009720955975353718, 0.03158125653862953, 0.09247038513422012, 0.048808369785547256, 0.008174203336238861, 0.031708307564258575, 0.02787904627621174, 0.14128026366233826, 0.0302340779453516, 0.03392859548330307, 0.03171432018280029, 0.10725171118974686, 0.0, 0.0], [0.808469295501709, 0.0056702690199017525, 0.006755992770195007, 0.008696584962308407, 0.010473440401256084, 0.003464736510068178, 0.006760997697710991, 0.0025889205280691385, 0.015097705647349358, 0.01603602059185505, 0.011288060806691647, 0.017277495935559273, 0.00779919745400548, 0.07962135970592499, 0.0], [0.02939111366868019, 0.0016060526249930263, 0.06005561351776123, 0.008175727911293507, 0.8099164366722107, 0.00039796155760996044, 0.0028282543644309044, 0.0033632616978138685, 0.0001569116284372285, 0.0036459483671933413, 0.05494074895977974, 0.001666039228439331, 0.015368940308690071, 5.016689101466909e-05, 0.008436810225248337]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9124024510383606, 0.08759760111570358, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.97407466173172, 0.01434413529932499, 0.011581187136471272, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8240247964859009, 0.03700513392686844, 0.06643965095281601, 0.07253041118383408, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5370795726776123, 0.15483324229717255, 0.0903128981590271, 0.12131187319755554, 0.0964624434709549, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9388002157211304, 0.028733234852552414, 0.0003431019140407443, 0.010481251403689384, 0.000839930260553956, 0.02080223336815834, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6318807005882263, 0.059302832931280136, 0.03517911583185196, 0.0317409411072731, 0.1174689382314682, 0.06869616359472275, 0.05573126673698425, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7625216841697693, 0.04435169696807861, 0.010552966967225075, 0.034276288002729416, 0.014203979633748531, 0.012703591957688332, 0.07259687036275864, 0.048792947083711624, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5725991725921631, 0.12681783735752106, 0.03727060556411743, 0.0431414395570755, 0.04129580408334732, 0.046447426080703735, 0.029164383187890053, 0.03751189261674881, 0.06575141847133636, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.12316936254501343, 0.026347072795033455, 0.27960509061813354, 0.041989635676145554, 0.44105827808380127, 0.006138973869383335, 0.007793003693223, 0.007175281643867493, 0.011286810971796513, 0.055436521768569946, 0.0, 0.0, 0.0, 0.0, 0.0], [0.44855940341949463, 0.1432485729455948, 0.010774216614663601, 0.07855041325092316, 0.01404787041246891, 0.014864671975374222, 0.008633698336780071, 0.0030146560166031122, 0.012792127206921577, 0.16408587992191315, 0.10142850130796432, 0.0, 0.0, 0.0, 0.0], [0.10228201746940613, 0.0238832775503397, 0.055250268429517746, 0.0520680733025074, 0.5759841799736023, 0.0011948152678087354, 0.0013662497512996197, 0.0025677781086415052, 0.0018699930515140295, 0.043160032480955124, 0.10887196660041809, 0.03150142729282379, 0.0, 0.0, 0.0], [0.39313194155693054, 0.04406267777085304, 0.04009911045432091, 0.07407406717538834, 0.04061643034219742, 0.007346798665821552, 0.009507199749350548, 0.013865168206393719, 0.0640539675951004, 0.04894229769706726, 0.0627753809094429, 0.0999651551246643, 0.10155977308750153, 0.0, 0.0], [0.7573198676109314, 0.015095721930265427, 0.007350177504122257, 0.013156400062143803, 0.005726401228457689, 0.008556416258215904, 0.008282607421278954, 0.010307075455784798, 0.002904881490394473, 0.017329057678580284, 0.015188980847597122, 0.0796574279665947, 0.030244464054703712, 0.02888045459985733, 0.0], [0.08579524606466293, 0.023773644119501114, 0.028759371489286423, 0.050303805619478226, 0.674644947052002, 0.0010763936443254352, 0.0011514866491779685, 0.002721975091844797, 0.0005617713322862983, 0.039091579616069794, 0.04861969128251076, 0.015520088374614716, 0.0040845321491360664, 0.00043291584006510675, 0.023462524637579918]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9979304075241089, 0.002069620881229639, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9971864819526672, 0.0010516609763726592, 0.0017618348356336355, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9594857096672058, 0.001310725579969585, 0.03694308176636696, 0.002260456094518304, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9891613125801086, 0.001052629784680903, 0.0048541901633143425, 0.001104357186704874, 0.003827564185485244, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9963598847389221, 0.0007981774397194386, 0.0007764195324853063, 0.00019262291607446969, 0.0002416159404674545, 0.0016313291853293777, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.981286346912384, 0.001863775309175253, 0.006344134453684092, 0.0002960922720376402, 0.004604941233992577, 0.0013966941041871905, 0.00420788861811161, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9950011372566223, 0.0013230532640591264, 0.0007920759962871671, 0.00024644460063427687, 0.0003405151073820889, 0.00016826140927150846, 0.0002837753272615373, 0.0018447580514475703, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9920291900634766, 0.0013539871433749795, 0.0007168339216150343, 9.214139572577551e-05, 0.00013419234892353415, 0.00019706363673321903, 0.00035278062568977475, 0.0002426079590804875, 0.004881155211478472, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5483196377754211, 0.0075838202610611916, 0.28265973925590515, 0.0029685653280466795, 0.14815661311149597, 0.0008172960951924324, 0.0006479909643530846, 0.0017078499076887965, 0.004129356704652309, 0.003009046893566847, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8642559051513672, 0.0036493062507361174, 0.004925842396914959, 0.06957440823316574, 0.038683053106069565, 0.009609837085008621, 0.0008548588375560939, 0.0005643004551529884, 0.003707138355821371, 0.0013952680164948106, 0.002780066104605794, 0.0, 0.0, 0.0, 0.0], [0.30252957344055176, 0.006584456656128168, 0.1401754468679428, 0.030065739527344704, 0.4806697964668274, 0.0005923578282818198, 0.0005309724947437644, 0.0024134721606969833, 0.008505754172801971, 0.0013766245683655143, 0.023764921352267265, 0.0027908117044717073, 0.0, 0.0, 0.0], [0.7773900032043457, 0.003722917754203081, 0.022378109395503998, 0.015139483846724033, 0.017408354207873344, 0.002416662173345685, 0.000709561922121793, 0.0007399603491649032, 0.13850137591362, 0.0023606533650308847, 0.005776867736130953, 0.0028014343697577715, 0.010654616169631481, 0.0, 0.0], [0.9735962152481079, 0.0012812522472813725, 0.002617918187752366, 9.891873924061656e-05, 0.0005098494002595544, 0.00012000725837424397, 0.00045229491661302745, 0.00010779645526781678, 0.002954537281766534, 0.001644388772547245, 0.0012141242623329163, 0.00018637391622178257, 0.0008510759216733277, 0.01436527632176876, 0.0], [0.0990300253033638, 0.000971083587501198, 0.06413238495588303, 0.005246965680271387, 0.8110400438308716, 8.250321843661368e-05, 6.596777529921383e-05, 0.0011991349747404456, 0.0008852760074660182, 0.00024891181965358555, 0.010336406528949738, 7.952339365147054e-05, 0.0049433172680437565, 0.00028918671887367964, 0.0014492359478026628]]], "attentionHeadNames": ["L10H7", "L11H10", "L9H9"], "tokens": ["<|endoftext|>", "When", " John", " and", " Mary", " went", " to", " the", " shops", ",", " John", " gave", " the", " bag", " to"]}
     )
@@ -1955,11 +1955,11 @@ <h3>Early Heads are Induction Heads(?!)<a class="headerlink" href="#Early-Heads-
 </pre></div>
 </div>
 <div class="output_area rendered_html docutils container">
-<div style='max-width: 800px;'><h2>Induction Heads</h2><br/><div id="circuits-vis-4f5c8cd3-096b" style="margin: 15px 0;"/>
+<div style='max-width: 800px;'><h2>Induction Heads</h2><br/><div id="circuits-vis-3078e839-768c" style="margin: 15px 0;"/>
     <script crossorigin type="module">
     import { render, AttentionHeads } from "https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js";
     render(
-      "circuits-vis-4f5c8cd3-096b",
+      "circuits-vis-3078e839-768c",
       AttentionHeads,
       {"attention": [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9995203018188477, 0.0004796621506102383, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9989193677902222, 0.00011719338363036513, 0.0009634991874918342, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.999607503414154, 4.1751700337044895e-05, 9.65828076004982e-06, 0.00034111665445379913, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9989573955535889, 3.091409962507896e-05, 1.212368533742847e-05, 0.0002459374663885683, 0.0007535542827099562, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9994303584098816, 3.2271585951093584e-05, 2.162250348192174e-05, 0.000438225019024685, 4.1958148358389735e-05, 3.5627250326797366e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.998076319694519, 1.6859989045769908e-05, 2.043985659838654e-06, 6.719817611156031e-05, 1.8109081167949626e-07, 0.00016430420510005206, 0.0016730953939259052, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9989305138587952, 0.0001340252347290516, 1.5985064237611368e-05, 0.000491619051899761, 6.565183241491468e-08, 6.484094228653703e-06, 1.1354626622051e-05, 0.00040989238186739385, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9949487447738647, 0.0005971402861177921, 1.4574082342733163e-05, 0.0018354525091126561, 9.743422424435266e-07, 7.463527435902506e-05, 5.5863079069240484e-06, 0.00020289771782699972, 0.002319993684068322, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9855782985687256, 6.499102164525539e-05, 8.181475277524441e-05, 0.0024699484929442406, 4.049685031759509e-08, 3.498060686979443e-05, 0.001560866367071867, 0.0018861292628571391, 0.00011740217451006174, 0.008205565623939037, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9950633645057678, 0.00021360788377933204, 0.00014613209350500256, 0.00030726491240784526, 2.292262615810614e-05, 0.0002385093830525875, 1.9861789041897282e-05, 0.000960799225140363, 0.0001365413481835276, 0.0016751259099692106, 0.001215968164615333, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9907191395759583, 0.00013381309690885246, 3.014638878084952e-06, 0.0023974282667040825, 2.1563739210250787e-05, 0.0004241704009473324, 1.383451308356598e-05, 5.452497134683654e-05, 2.246501207991969e-05, 0.0017353435978293419, 0.0022565408144146204, 0.002218238776549697, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.99763023853302, 7.048921543173492e-05, 1.2420621260389453e-06, 6.923218461452052e-05, 0.0007476441096514463, 0.000763593299780041, 7.526726676587714e-06, 7.549879228463396e-05, 2.769195930341084e-07, 9.996434528147802e-05, 3.433594974922016e-05, 0.00019309873459860682, 0.00030683254590258, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9413580298423767, 0.0009730295860208571, 0.0004518784990068525, 0.0012102690525352955, 2.08603341889102e-06, 0.0018085278570652008, 1.965607407328207e-05, 0.03966164216399193, 0.0004176551301497966, 0.003035214263945818, 0.002604098990559578, 0.00038386619416996837, 0.007335478439927101, 0.0007384858326986432, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.98450767993927, 0.00010836195724550635, 2.4642587959533557e-05, 0.0004206285811960697, 1.2694325732809375e-06, 0.00011915803042938933, 9.773763304110616e-05, 0.0023661903105676174, 0.00036190610262565315, 0.003964339382946491, 0.0009553474374115467, 0.0006011889781802893, 0.002696628449484706, 0.00010731287329690531, 0.003667621174827218, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9236841797828674, 0.0004220438713673502, 1.1906979125342332e-05, 0.036959629505872726, 0.00012378330575302243, 0.0005373558960855007, 0.0029662158340215683, 0.0006545133655890822, 5.279582546791062e-05, 0.0077215940691530704, 0.0131848668679595, 7.147376891225576e-05, 0.00951047521084547, 0.0007938834605738521, 0.0006112216506153345, 0.0026941129472106695, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9707465171813965, 7.250557973748073e-05, 4.7150632553893956e-07, 0.0007667056052014232, 1.613832296243345e-06, 4.375730350147933e-05, 0.00013073488662485033, 0.0010341654997318983, 0.000541800691280514, 0.006641003768891096, 0.014592697843909264, 0.00018458777049090713, 0.003935785032808781, 3.716452511071111e-06, 2.8841026505688205e-05, 0.0008475544163957238, 0.0004274799139238894, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9573933482170105, 0.00019599858205765486, 2.3228804479913379e-07, 0.003389731515198946, 9.590152103555738e-07, 5.782760854344815e-05, 1.3954500900581479e-05, 9.882948506856337e-05, 0.00014849765284452587, 0.006945967674255371, 0.01562657207250595, 8.081036867224611e-06, 0.011182066984474659, 6.349627255985979e-06, 2.5934892619261518e-05, 0.00012999530008528382, 0.0007364018238149583, 0.004039338324218988, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.990619421005249, 6.466358172474429e-05, 9.901318662741687e-07, 0.00021274133177939802, 1.4796115465287585e-05, 0.00017918922821991146, 6.778116585337557e-06, 7.372434629360214e-05, 0.0002996290859300643, 0.0013087284751236439, 0.0005674119456671178, 0.0011630650842562318, 0.0028649545274674892, 9.635516471462324e-05, 8.433945913566276e-05, 0.00028768807533197105, 4.3501222535269335e-05, 0.00038704575854353607, 0.001724953530356288, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9956457018852234, 2.669214882189408e-05, 3.9460044121142346e-08, 0.001181368250399828, 1.471557879995089e-05, 0.0001445170200895518, 0.0004370114766061306, 3.213761010556482e-05, 8.180093402643251e-08, 9.59495737333782e-05, 0.0007029082044027746, 1.2536466783785727e-05, 0.00018710187578108162, 0.00011147692566737533, 7.006971713963139e-07, 4.411066583998036e-06, 0.0007388183148577809, 4.973216618964216e-06, 9.905266779242083e-05, 0.0005597113631665707, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9956768155097961, 3.464311521383934e-05, 5.037085770709382e-08, 0.00012908628559671342, 3.5288087474327767e-06, 8.07559808890801e-06, 3.11900585074909e-05, 6.715198833262548e-05, 1.5350008197856368e-07, 0.00010576295608188957, 0.0004207944730296731, 4.739598443848081e-05, 0.00017379986820742488, 3.910295163223054e-06, 3.7129705106053734e-06, 6.050187221262604e-05, 4.920005812891759e-05, 4.588661977322772e-05, 0.0002989978529512882, 0.00027882878202944994, 0.0025604318361729383, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9873839616775513, 0.002735754009336233, 1.0963850400003139e-05, 0.0005862272810190916, 6.101510280132061e-06, 6.327711162157357e-05, 2.1911782823735848e-05, 0.00033786968560889363, 6.588418182218447e-05, 0.0013113167369738221, 0.0019715256057679653, 0.00013529015996027738, 0.0018900128779932857, 0.00018075927800964564, 0.00026106255245395005, 0.00037814932875335217, 9.485983173362911e-05, 4.87527868244797e-05, 0.00035683103487826884, 0.0007243757136166096, 0.00010285121243214235, 0.0013322114245966077, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3808109164237976, 0.005228960886597633, 0.6019217371940613, 0.006041430402547121, 3.194508099113591e-05, 4.269973578630015e-05, 8.645826892461628e-05, 0.0006069993833079934, 0.0006299586966633797, 0.0009580003097653389, 0.0006117315497249365, 0.0002489396429155022, 0.0002464656427036971, 6.495700654340908e-05, 0.0006897134589962661, 0.00021990980894770473, 7.002806523814797e-05, 0.00014192514936439693, 2.700363074836787e-05, 5.169477663002908e-05, 4.5723249968432356e-06, 0.00029897037893533707, 0.0009650306892581284, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.563797116279602, 0.0085763493552804, 0.004927160684019327, 0.34193748235702515, 2.1189502149354666e-05, 0.00020753320131916553, 0.0003593186556827277, 0.0014688574010506272, 0.0003070282400585711, 0.02934207208454609, 0.004261575639247894, 8.383052772842348e-06, 0.0024304059334099293, 9.930188389262185e-05, 0.000542300462257117, 9.130241960519925e-05, 0.03218064829707146, 7.990232552401721e-05, 0.0003810952475760132, 0.00010837644367711619, 3.33801144734025e-05, 0.0008215161506086588, 0.0017809885321184993, 0.006236794404685497, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3740951418876648, 0.00012492266250774264, 8.869202247296926e-06, 0.0009476605337113142, 0.6135826706886292, 0.002516972366720438, 0.0002441502292640507, 5.642483665724285e-05, 1.795958496586536e-06, 0.0004439418844413012, 5.296186645864509e-05, 0.0001609584578545764, 0.0007701200665906072, 0.005056836176663637, 1.382048776576994e-05, 5.882297773496248e-05, 0.00018946161435451359, 0.00012655068712774664, 3.178522456437349e-05, 6.542137998621911e-05, 1.7726270016282797e-05, 0.0008768205880187452, 0.00038724290789105, 9.015519140120887e-07, 0.00016804093320388347, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.18932704627513885, 0.0003522285260260105, 1.1454378181952052e-05, 0.0007539974758401513, 0.003070831298828125, 0.7761431336402893, 5.7844576076604426e-05, 0.004086362663656473, 1.6754691387177445e-05, 0.0018927170895040035, 0.00031636765925213695, 0.0005395743064582348, 0.005218177568167448, 0.007962456904351711, 0.004062105901539326, 0.002489584032446146, 1.4082329471420962e-05, 1.4990718227636535e-05, 0.00015581185289192945, 0.0003996991435997188, 5.5086038628360257e-05, 0.0019400393357500434, 0.0003281844255980104, 1.4123389746600878e-07, 8.962309038906824e-06, 0.0007822647457942367, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.02010822296142578, 2.635493547131773e-05, 2.7333503567206208e-06, 0.0002452948538120836, 1.4769997505936772e-05, 2.0045878045493737e-05, 0.9765581488609314, 0.0007374248816631734, 2.2332008029479766e-06, 0.00016837129078339785, 0.00137846521101892, 1.0426949756947579e-06, 9.273762407246977e-05, 6.001193469273858e-06, 1.852532113844063e-05, 7.490423013223335e-05, 2.892227712436579e-05, 4.0283404814545065e-05, 1.959171459020581e-05, 2.944982952612918e-05, 8.47621322463965e-06, 0.0002561432193033397, 5.941176277701743e-05, 2.014576523379219e-07, 7.743915921309963e-05, 7.494620604120428e-06, 1.74276083271252e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.22277945280075073, 0.002593980636447668, 2.9011016522417776e-06, 0.0005522919236682355, 1.904411533359962e-06, 0.0006727795698679984, 0.0021306686103343964, 0.6761497855186462, 6.977441080380231e-05, 0.003776289988309145, 0.06293372064828873, 0.0015865935711190104, 0.0017082190606743097, 3.881497832480818e-05, 0.009209989570081234, 0.0013166061835363507, 0.00018038915004581213, 0.001267515355721116, 0.0018657379550859332, 0.0009925103513523936, 3.7589063140330836e-05, 0.006426853593438864, 0.0020985312294214964, 3.3772406027310353e-07, 2.868737283279188e-05, 2.698123182653944e-07, 0.00025180247030220926, 0.001325962133705616, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08230321109294891, 0.0005252842092886567, 4.260616606188705e-06, 0.0006264003459364176, 8.583574384601889e-08, 4.224356871418422e-06, 4.716531293524895e-06, 0.0003961432375945151, 0.8586453795433044, 0.05151975154876709, 0.0026339711621403694, 1.6450698240078054e-05, 0.0006709631998091936, 1.5013217762316344e-06, 9.040276927407831e-05, 7.098149944795296e-05, 7.727313459326979e-06, 0.00020618800772354007, 0.0005295966984704137, 0.0013012847630307078, 5.2623075674773645e-08, 9.972530097002164e-05, 0.00018814747454598546, 2.380612386332359e-06, 6.255433254409581e-05, 3.8025717685741256e-08, 9.557845714880386e-07, 3.836729320028098e-06, 8.390715811401606e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.465234637260437, 0.004093279130756855, 6.743345238646725e-06, 0.004121774807572365, 5.443537247629138e-07, 5.419518129201606e-05, 9.294981850871409e-07, 0.00026017383788712323, 0.00907624140381813, 0.5008078813552856, 0.0023183668963611126, 2.4238510377472267e-05, 0.0030431346967816353, 4.68218931928277e-05, 0.00012087374489055946, 0.0001290314394282177, 0.00011785242531914264, 4.014383375761099e-05, 0.0024619202595204115, 0.0034230302553623915, 1.7263497511521564e-06, 0.0002791083534248173, 0.002281592693179846, 4.889936462859623e-06, 0.00025089550763368607, 2.8178769184705743e-07, 1.050695664162049e-05, 1.070987195816997e-06, 4.786956196767278e-05, 0.0017402973026037216, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3585144281387329, 0.0011027655564248562, 0.00016592990141361952, 0.013357667252421379, 2.546785253798589e-07, 6.185192614793777e-05, 0.0057292357087135315, 0.010611956007778645, 0.0007551729213446379, 0.06400889158248901, 0.4949103593826294, 8.391502342419699e-05, 0.005665580276399851, 2.122150362993125e-06, 0.0002814042672980577, 0.0004684626474045217, 0.0004158185620326549, 0.0012447141343727708, 0.012713258154690266, 0.0010433174902573228, 1.1126050594612025e-05, 0.0011420734226703644, 0.0006368785980157554, 7.042537617962807e-05, 0.002744977129623294, 1.2284903050385765e-07, 1.483806499891216e-05, 0.005110186990350485, 0.004603618290275335, 0.00015696643094997853, 0.014371593482792377, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.387024849653244, 0.0012950673699378967, 4.0595266909804195e-05, 0.0016053735744208097, 5.915556903346442e-05, 0.00023957251687534153, 2.72913839580724e-05, 0.004568422678858042, 0.0004761815653182566, 0.01562541350722313, 0.011085761711001396, 0.49522680044174194, 0.03870179131627083, 0.00026717083528637886, 0.0020083452109247446, 0.004042898770421743, 0.00032641590223647654, 0.0036371152382344007, 0.005066386889666319, 0.003996914252638817, 5.871349458175246e-06, 0.012405160814523697, 0.004702437669038773, 1.1574948075576685e-05, 0.0001442672364646569, 1.3832131116942037e-05, 7.560355879832059e-05, 1.5469962818315253e-05, 0.0020267819054424763, 3.810078851529397e-05, 0.003570300992578268, 0.001669161138124764, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4725484848022461, 0.0012542733456939459, 1.6021392639231635e-06, 0.007470476441085339, 2.098127151839435e-05, 0.00022495862504001707, 3.144747097394429e-06, 0.00015600249753333628, 5.6978915381478146e-05, 0.008746420964598656, 0.006501034367829561, 0.005835692398250103, 0.4674169719219208, 0.0013900346821174026, 0.0022286404855549335, 0.0015983363846316934, 0.0009148887475021183, 4.253075894666836e-05, 0.007089768536388874, 0.002937783719971776, 9.027125088323373e-06, 0.002526575466617942, 0.004196966998279095, 9.68642325460678e-07, 0.0005582970916293561, 1.858499126683455e-05, 8.944844012148678e-05, 6.846757969469763e-06, 7.68072932260111e-05, 1.4904732779541519e-05, 0.0030944147147238255, 0.0010494303423911333, 0.0019187491852790117, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14304865896701813, 0.0001688210031716153, 5.145543582329992e-07, 0.00013525491522159427, 0.001670985366217792, 0.000564366695471108, 4.049839844810776e-06, 4.1035786125576124e-05, 6.743645712958823e-07, 0.00026636559050530195, 3.54247895302251e-05, 0.00038403936196118593, 0.001104785711504519, 0.8484564423561096, 0.00022315295063890517, 0.00125994929112494, 2.182349817303475e-05, 4.0234138509731565e-07, 9.149295692623127e-06, 7.982308306964114e-05, 3.274861228419468e-05, 0.000818326196167618, 0.0002738100884016603, 6.036240307594198e-08, 5.658039754052879e-06, 0.0007130159065127373, 0.00026146037271246314, 3.5881919302482856e-06, 2.1928895876044407e-05, 4.159101507639207e-08, 0.00011863742111017928, 1.6391287545047817e-06, 3.8121448596939445e-05, 0.00023530120961368084, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.24328209459781647, 0.0009063901379704475, 0.0002717962197493762, 0.0012521950993686914, 3.105668383796001e-06, 0.001761557999998331, 7.013605227257358e-06, 0.040556132793426514, 0.0003538632008712739, 0.003045480465516448, 0.0016025315271690488, 0.0006016580737195909, 0.009119357913732529, 0.00086904939962551, 0.6830158829689026, 0.002626276109367609, 0.00010224885045317933, 7.45571151128388e-06, 3.991548510384746e-05, 0.00021531074889935553, 2.0861371012870222e-06, 0.000726094760466367, 0.0004009866388514638, 1.702951885818038e-05, 6.652241427218542e-05, 3.073251377827546e-07, 0.00018005711899604648, 5.3525491239270195e-06, 0.0071047646924853325, 2.1172911147004925e-05, 0.00044749100925400853, 5.941993367741816e-05, 3.588023173506372e-05, 0.001178023056127131, 0.00011552044452400878, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.24815866351127625, 0.0004487273108679801, 9.697681889520027e-06, 0.0010719134006649256, 7.626599654031452e-06, 0.00018865316815208644, 6.339164974633604e-05, 0.004560624714940786, 0.0010741765145212412, 0.015649856999516487, 0.002170345513150096, 0.0007331715896725655, 0.007097125519067049, 0.00044510516454465687, 0.016780385747551918, 0.6907860040664673, 0.00019302917644381523, 0.0003835348761640489, 0.0009248864953406155, 0.001862714416347444, 2.057407328948102e-07, 0.002524268114939332, 0.0015590769471600652, 3.3248329600610305e-07, 1.708475247141905e-05, 4.2712878212114447e-07, 1.1026169886463322e-05, 4.378564426588127e-06, 0.00015843719302210957, 1.5476502085221e-05, 0.0009950557723641396, 4.288491982151754e-05, 5.0623875722521916e-05, 0.000995215610601008, 1.3690095329366159e-05, 0.0010022438364103436, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15202470123767853, 0.0005997071275487542, 1.1835468285426032e-05, 0.10644718259572983, 9.087667422136292e-05, 0.0003228001296520233, 0.000542075838893652, 0.0006674103206023574, 0.00022456956503447145, 0.021209707483649254, 0.019741186872124672, 0.00012132841220591217, 0.015023781917989254, 0.0003238526696804911, 0.0003435326216276735, 0.003281805431470275, 0.6119738221168518, 0.00224235188215971, 0.008034387603402138, 0.0039560976438224316, 0.00143277901224792, 0.011238718405365944, 0.0009119117166846991, 1.6121168300742283e-05, 0.034571193158626556, 4.049971539643593e-05, 7.184680725913495e-05, 0.0005501421983353794, 0.0001563764235470444, 6.361229679896496e-06, 0.00147312565241009, 0.0009954210836440325, 1.2639467058761511e-05, 0.0010144341504201293, 3.702372487168759e-05, 5.788439739262685e-05, 0.00023058420629240572, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.035443101078271866, 2.0004812540719286e-05, 4.18805292667912e-08, 0.0003809734480455518, 5.874024395779998e-07, 2.792186933220364e-06, 2.425247657811269e-05, 8.103247819235548e-05, 0.00011144909512950107, 0.0027193769346922636, 0.00454584788531065, 4.083846943103708e-05, 0.0014098533429205418, 5.68071186535235e-07, 3.1567922178510344e-06, 8.66913323989138e-05, 0.0001365105708828196, 0.8487799763679504, 0.10322029143571854, 0.00040226749842986465, 2.244725919808843e-06, 0.0014588433550670743, 0.00012918819265905768, 6.671142216418957e-08, 7.290130452020094e-05, 3.444819185460801e-07, 1.4172115925248363e-06, 1.9379569494049065e-05, 4.5301294449018314e-05, 6.376938017638167e-06, 0.00026465157861821353, 0.0004582676920108497, 2.116967152687721e-05, 9.94484216789715e-05, 4.348984106172793e-08, 3.564729809113487e-07, 2.450852434776607e-06, 8.022138899832498e-06, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2318733036518097, 0.0003618353803176433, 8.543166529761947e-08, 0.006350396666675806, 1.0386469284640043e-06, 1.706758121144958e-05, 9.30641544982791e-06, 0.00014515469956677407, 9.0070599981118e-05, 0.01544187217950821, 0.04194100946187973, 8.23421487439191e-06, 0.022686535492539406, 5.4958513828751165e-06, 2.1405727238743566e-05, 5.323981531546451e-05, 0.0014441021485254169, 0.005718633532524109, 0.6479307413101196, 0.012128104455769062, 4.028283728985116e-05, 0.002520335605368018, 0.0007621925324201584, 6.570355992607801e-08, 0.0007668299949727952, 4.547823095890635e-07, 8.124479791149497e-06, 1.3828406736138277e-05, 3.526886212057434e-05, 9.165401934296824e-06, 0.0020918850786983967, 0.004286687355488539, 3.2024597658164566e-06, 0.0017294595018029213, 5.357091481528187e-07, 2.501844164726208e-06, 2.884956984416931e-06, 8.676738070789725e-05, 0.0014117915416136384, 0.0, 0.0, 0.0, 0.0], [0.3010721206665039, 0.0003067184588871896, 7.214547395051341e-07, 0.0010787947103381157, 3.671133526950143e-05, 9.958079317584634e-05, 2.7117800982523477e-06, 8.51582590257749e-05, 0.0006431526271626353, 0.00647402461618185, 0.0024339223746210337, 0.0026007171254605055, 0.012783723883330822, 0.0001047167825163342, 7.190534233814105e-05, 0.00023829020210541785, 0.00014132891374174505, 0.0010901358909904957, 0.006501613184809685, 0.6443015933036804, 0.0005654482520185411, 0.011171936988830566, 0.0017330633709207177, 3.266845851612743e-07, 7.267465116456151e-05, 1.726137998048216e-05, 5.252150003798306e-05, 2.8104516331950435e-06, 2.3941935069160536e-05, 6.633036537095904e-05, 0.002350717782974243, 0.00014581516734324396, 0.0007032529101707041, 0.0013972821179777384, 3.202367588528432e-05, 2.3009477445157245e-05, 2.68677049461985e-05, 1.7945263607543893e-05, 0.0001209545589517802, 0.001408109557814896, 0.0, 0.0, 0.0], [0.012865342199802399, 1.5182659808488097e-05, 2.972013524171757e-09, 0.0002875621721614152, 1.2528023489721818e-06, 1.252613128599478e-05, 4.250307028996758e-06, 8.040903594519477e-06, 3.2277103656497275e-08, 5.239708480075933e-05, 0.00013598549412563443, 1.002539192995755e-06, 5.5890355724841356e-05, 7.012691639829427e-06, 9.724845284608818e-08, 3.212419130704802e-07, 4.670393173000775e-05, 1.5471712231374113e-06, 1.7591864889254794e-05, 5.334270463208668e-05, 0.9805435538291931, 0.005806764122098684, 1.860846532508731e-05, 1.2037257857144823e-09, 3.0451696147792973e-05, 6.754650598850276e-07, 4.56516090707737e-06, 5.748487637902144e-06, 3.3398507639503805e-06, 7.534131096775809e-10, 7.689136509725358e-06, 3.901714080711827e-06, 4.307142731363456e-08, 5.192545700083429e-07, 5.106750791128434e-07, 8.534786211100709e-09, 1.0071753209217604e-08, 2.59315993389464e-06, 1.4836371065030107e-07, 3.5096545047963446e-07, 4.402238118927926e-06, 0.0, 0.0], [0.2408866286277771, 0.0003843706217594445, 3.8272382596460375e-08, 0.0007521198713220656, 4.429959517437965e-06, 1.3616883734357543e-05, 1.0370951713412069e-05, 0.00026089049060828984, 2.180447154387366e-06, 0.0008239771705120802, 0.0025784452445805073, 0.0001127499490394257, 0.001231508213095367, 3.884355010086438e-06, 3.183517401339486e-05, 0.00015905841428320855, 0.00016030347615014762, 0.00039731874130666256, 0.0017629768699407578, 0.001543086371384561, 0.0018108984222635627, 0.7393378019332886, 0.00752194132655859, 2.4824957645108725e-09, 2.4742143068579026e-05, 3.882949215494591e-07, 2.5054800971702207e-06, 3.208101816198905e-06, 4.296221959521063e-05, 2.8116325268001674e-08, 1.3948349078418687e-05, 1.6366040654247627e-05, 6.464315902121598e-06, 3.004602695000358e-05, 1.3071932869479497e-07, 9.608824029783136e-07, 1.787801124919497e-06, 1.753078095134697e-06, 4.386996351968264e-06, 1.6937889085966162e-05, 1.7175851098727435e-05, 2.5725426894496195e-05, 0.0], [0.5535919070243835, 0.17262260615825653, 5.0000453484244645e-05, 0.01042777020484209, 8.65629335748963e-05, 0.0002456592337694019, 5.955337837804109e-05, 0.001045247190631926, 0.0008384220418520272, 0.01357327215373516, 0.011254074051976204, 0.003105845768004656, 0.02095966413617134, 0.0010894862934947014, 0.0004093680181540549, 0.002362574217841029, 0.0016563499812036753, 0.0009357953676953912, 0.0033135719131678343, 0.00543051166459918, 0.0010150577872991562, 0.032745711505413055, 0.1570950448513031, 3.374255175003782e-05, 0.0007222488638944924, 2.940950980701018e-05, 6.13709562458098e-05, 3.093114719376899e-05, 0.00024038334959186614, 4.459061528905295e-05, 0.0006923413602635264, 0.00030414367211051285, 0.0001727685157675296, 0.0012000406859442592, 0.00010696896788431332, 2.8368263883749023e-05, 1.842521487560589e-05, 2.8223597837495618e-05, 3.744996865862049e-05, 0.00011934195208596066, 0.00018961371097248048, 3.498619480524212e-05, 0.0019905013032257557]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9940896034240723, 0.0059104301035404205, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9918997287750244, 0.002507372060790658, 0.005592965055257082, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.99346524477005, 0.003521271515637636, 0.00027059370768256485, 0.0027428686153143644, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9289718270301819, 0.003740765620023012, 0.0001203682622872293, 0.002464901190251112, 0.06470214575529099, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9885430335998535, 0.0006141006015241146, 0.00041145377326756716, 0.007392834406346083, 0.000539508939255029, 0.002499047899618745, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9694589376449585, 0.0009395341039635241, 2.8261547413421795e-05, 0.0020353952422738075, 2.71175304078497e-05, 0.00829475186765194, 0.019215956330299377, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9474747776985168, 0.005424594972282648, 0.004983103834092617, 0.021659083664417267, 1.2853307453042362e-05, 0.0011076473165303469, 0.001525420811958611, 0.01781250163912773, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9157270193099976, 0.006453890353441238, 0.0011921703116968274, 0.04077358543872833, 3.2812098652357236e-05, 0.002033841796219349, 0.00022136607731226832, 0.010999627411365509, 0.022565681487321854, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8299553990364075, 0.0021487579215317965, 0.002185288118198514, 0.04418088495731354, 1.725433139654342e-05, 0.0014081649715080857, 0.020800573751330376, 0.08762412518262863, 0.000955707801040262, 0.010723880492150784, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9581490159034729, 0.004779844544827938, 0.00035515165654942393, 0.0029918320942670107, 0.0032907268032431602, 0.007958430796861649, 0.0017856257036328316, 0.013284464366734028, 0.00015784945571795106, 0.0009225388057529926, 0.0063244919292628765, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.944205641746521, 0.0028841965831816196, 4.829783574678004e-05, 0.007261567283421755, 0.000468916492536664, 0.005381070543080568, 0.00017496546206530184, 0.006957130506634712, 0.00012532716209534556, 0.004074090626090765, 0.016885550692677498, 0.011533298529684544, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7170465588569641, 0.0029263142496347427, 1.372535371046979e-05, 0.0010318169370293617, 0.1216491237282753, 0.14445394277572632, 0.0009355274378322065, 0.005332366097718477, 4.7773132791917305e-06, 0.0012480594450607896, 0.00030511102522723377, 0.0019744164310395718, 0.0030781622044742107, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7894357442855835, 0.0020973754581063986, 0.0020318119786679745, 0.010099953971803188, 0.00036911203642375767, 0.06713488698005676, 0.001530290232039988, 0.08991599082946777, 0.0002695393341127783, 0.005345543846487999, 0.005062697920948267, 0.0006220968207344413, 0.019640257582068443, 0.006444734986871481, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8822957873344421, 0.0018603845965117216, 0.00040732327033765614, 0.003660863032564521, 0.0005736579769290984, 0.007988521829247475, 0.02257135882973671, 0.02860121801495552, 0.0003292130131740123, 0.0042566233314573765, 0.006161138415336609, 0.0010443199425935745, 0.02063308097422123, 0.005851183086633682, 0.013765350915491581, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.786965548992157, 0.0042131864465773106, 0.0001677500840742141, 0.06983140110969543, 0.002818087814375758, 0.007715716026723385, 0.031199535354971886, 0.017051437869668007, 0.00018332427134737372, 0.012005697004497051, 0.017662225291132927, 0.0002890100877266377, 0.03032497689127922, 0.004889386706054211, 0.003095801919698715, 0.011587009765207767, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9286779761314392, 0.0015812769997864962, 1.4040829228179064e-06, 0.0019428395899012685, 1.1035229363187682e-05, 0.0002707966777961701, 0.002038027858361602, 0.0036648765671998262, 0.00022354313114192337, 0.0007879888289608061, 0.04505370557308197, 0.00048093145596794784, 0.010849039070308208, 3.6126039049122483e-06, 6.594638398382813e-05, 0.0008642339380457997, 0.0034828479401767254, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8384721875190735, 0.0026208001654595137, 3.342468335176818e-05, 0.01232653483748436, 7.991644088178873e-05, 0.003431107848882675, 0.0022304044105112553, 0.008115601725876331, 0.0006596868042834103, 0.008109694346785545, 0.0638236477971077, 0.00041881605284288526, 0.03681996464729309, 0.00012104758206987754, 0.0005988333723507822, 0.00862804614007473, 0.003624672070145607, 0.00988560076802969, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8221591114997864, 0.0057971584610641, 0.00019727385370060802, 0.005305740050971508, 0.0009490191005170345, 0.01202884316444397, 0.00190554594155401, 0.0023991470225155354, 0.01059637125581503, 0.014539752155542374, 0.011935291811823845, 0.01170417945832014, 0.048201464116573334, 0.006271726917475462, 0.006691561546176672, 0.02466079778969288, 0.005656811874359846, 0.005013483576476574, 0.003986795898526907, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9537661671638489, 0.0015298646176233888, 5.0040102905768435e-06, 0.007204053923487663, 0.0002875779173336923, 0.0017434227047488093, 0.00982952956110239, 0.004322257358580828, 3.13615760205721e-06, 0.00010985424887621775, 0.008465062826871872, 0.0003484692715574056, 0.0013010905822739005, 0.0008953196229413152, 6.606405804632232e-05, 5.8244528190698475e-05, 0.0037666442804038525, 0.0005685480427928269, 0.001456835656426847, 0.004272699821740389, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9650986790657043, 0.0014334715669974685, 4.691014055424603e-06, 0.0012125533539801836, 0.0003355584340170026, 0.0012382488930597901, 0.0013702923897653818, 0.002564979949966073, 6.760245128134557e-07, 7.471856224583462e-05, 0.001875573769211769, 0.0005754472804255784, 0.0028297884855419397, 0.0007055316236801445, 0.00027714675525203347, 0.00048612672253511846, 0.001569450250826776, 0.0003992257115896791, 0.0028444859199225903, 0.004788658116012812, 0.010314569808542728, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8593109250068665, 0.020267227664589882, 0.002777141286060214, 0.013781089335680008, 0.00030035056988708675, 0.002284612273797393, 0.0010426462395116687, 0.009922247380018234, 0.003133647609502077, 0.007885495200753212, 0.009642801247537136, 0.001854671980254352, 0.00787472166121006, 0.0011870941380038857, 0.002824161434546113, 0.00242280843667686, 0.004221721086651087, 0.004099525045603514, 0.006750837899744511, 0.0058750067837536335, 0.002170956926420331, 0.030370241031050682, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10544665902853012, 0.01270485483109951, 0.7639007568359375, 0.08842571824789047, 0.00018797894881572574, 0.0002628951915539801, 0.0007833988056518137, 0.001709726988337934, 0.002409537322819233, 0.014530741609632969, 0.0006739442469552159, 3.582143835956231e-05, 0.0008068872266449034, 0.000150876454426907, 0.001663832925260067, 0.0002909536415245384, 0.001969669247046113, 2.807584314723499e-05, 8.648991206428036e-05, 0.0001831684639910236, 5.4813881433801726e-05, 0.0004074028111062944, 0.003285784274339676, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.33122679591178894, 0.007398602087050676, 0.006046590860933065, 0.38090354204177856, 0.0007559282821603119, 0.003547786269336939, 0.00850890576839447, 0.02200973406434059, 0.0006401475402526557, 0.06486588716506958, 0.012087974697351456, 2.1607689632219262e-05, 0.004766723141074181, 0.0004923217929899693, 0.002488807076588273, 0.00041253515519201756, 0.11648410558700562, 0.00017437727365177125, 0.001129980431869626, 0.0012968714581802487, 0.001893762033432722, 0.006374640390276909, 0.012117794714868069, 0.014354660175740719, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.022601181641221046, 0.0004386410873848945, 2.7166800009581493e-06, 0.0003694462648127228, 0.9364854693412781, 0.032159216701984406, 0.00016640119429212064, 0.00037730683106929064, 2.719356189118116e-07, 9.796216181712225e-05, 2.4832312192302197e-05, 0.00014765629020985216, 0.0004301169828977436, 0.004391680471599102, 0.0003911138919647783, 5.064404831500724e-05, 0.00018704244575928897, 6.401351129170507e-06, 2.0583942387020215e-05, 0.0002504437288735062, 4.058926060679369e-05, 0.0006710129673592746, 0.0006265917327255011, 2.0016514099552296e-06, 6.063750697649084e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.039748288691043854, 0.0016230319160968065, 2.9864083899155958e-06, 0.0004959744983352721, 0.0075991530902683735, 0.903511106967926, 0.00030855785007588565, 0.020541049540042877, 3.7260215322021395e-06, 0.0009059454896487296, 0.0005234951386228204, 0.00019036592857446522, 0.002769604790955782, 0.001758625265210867, 0.006682183127850294, 0.0016490903217345476, 0.00027723185485228896, 2.3619986677658744e-05, 0.0003351525228936225, 0.001790589652955532, 0.00015660120698157698, 0.0019276170060038567, 0.0019798888824880123, 1.2426404509824351e-06, 5.812747258460149e-05, 0.005136688239872456, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.017394619062542915, 8.34300517453812e-05, 1.299515815844643e-06, 0.0006728374864906073, 1.4733542229805607e-05, 0.00010638036474119872, 0.9528148770332336, 0.021972037851810455, 7.827152899153589e-07, 1.5493780665565282e-05, 0.0031969614792615175, 3.257389471400529e-05, 0.0001382468908559531, 2.389342444075737e-05, 0.000840842432808131, 0.00010894631850533187, 0.00028938756440766156, 0.0001089509969460778, 6.79441072861664e-05, 0.00012526210048235953, 0.00019726519531104714, 0.0008631302043795586, 0.0002980587596539408, 2.3374896045424975e-06, 0.0003466096241027117, 2.3777269234415144e-05, 0.00025927400565706193, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05558102950453758, 0.0006445589242503047, 4.0934705225481594e-07, 0.00042194841080345213, 1.340792550763581e-06, 0.000646521570160985, 0.0018358062952756882, 0.9107965230941772, 6.180139735079138e-06, 0.00014045907300896943, 0.010802187956869602, 0.0002091953792842105, 0.0011495650978758931, 1.2848360711359419e-05, 0.003265448845922947, 0.0001199487887788564, 0.00020614646200556308, 0.0006146047380752861, 0.0012190378038212657, 0.001267205341719091, 0.00038551114266738296, 0.0035668814089149237, 0.0011672672117128968, 5.425858375929238e-07, 0.0003176873433403671, 1.1132586905659991e-06, 0.002803834853693843, 0.002816123189404607, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08360803127288818, 0.0049411761574447155, 5.052056076237932e-05, 0.007228298112750053, 1.1925421858904883e-06, 0.000162318188813515, 0.0002728418621700257, 0.005935708060860634, 0.6047206521034241, 0.26288917660713196, 0.005585616920143366, 7.11743050487712e-05, 0.0013994898181408644, 2.673692279131501e-06, 0.0015110736712813377, 9.4614464615006e-05, 0.0003633210726547986, 0.0007099778158590198, 0.0009741897229105234, 0.003578288946300745, 1.0970221410389058e-05, 0.0001563943224027753, 0.003689345670863986, 0.0002846125280484557, 0.004334022291004658, 1.3892353081246256e-06, 0.00024127897631842643, 0.0001852042623795569, 0.006996475625783205, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1152101382613182, 0.006714800372719765, 0.000162112366524525, 0.0255129411816597, 6.206613761605695e-06, 0.0004051269206684083, 5.4560387070523575e-05, 0.00351078063249588, 0.014202400110661983, 0.7775508165359497, 0.0034905581269413233, 5.7819983339868486e-05, 0.0047684297896921635, 3.606372411013581e-05, 0.0007714288076385856, 0.0003429505741223693, 0.002843409776687622, 9.518005390418693e-05, 0.003136677900329232, 0.005728919059038162, 4.441967757884413e-05, 0.0004939687205478549, 0.008516553789377213, 0.0005445042625069618, 0.014522203244268894, 5.894724836252863e-06, 0.00030799367232248187, 2.679031968000345e-05, 0.0010097086196765304, 0.009926472790539265, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.03713580593466759, 0.0008775312453508377, 7.848548557376489e-05, 0.015299011021852493, 1.4544053783538402e-06, 0.00010877755266847089, 0.006165174767374992, 0.014768626540899277, 7.875503069953993e-05, 0.001974995480850339, 0.845594584941864, 7.480713975382969e-05, 0.0028255584184080362, 5.115834369462391e-07, 0.00017026669229380786, 0.00015173685096669942, 0.006004323251545429, 0.0034732050262391567, 0.012612894177436829, 0.0006110360845923424, 0.0005419608787633479, 0.0009975193534046412, 0.002906821435317397, 0.0003956863656640053, 0.020268796011805534, 1.613207473383227e-06, 0.00021117184951435775, 0.007802305277436972, 0.017504550516605377, 2.8824499167967588e-05, 0.0013330691726878285, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04171165078878403, 0.0007273957598954439, 5.084281838207971e-06, 0.0004205020668450743, 0.0002682959020603448, 0.00036437364178709686, 0.00021653271687682718, 0.0014350030105561018, 5.140026132721687e-06, 9.150908590527251e-05, 0.0012519678566604853, 0.9164962768554688, 0.025602256879210472, 0.0013075522147119045, 0.0030348259024322033, 0.0005533328512683511, 0.00034456001594662666, 0.00023525662254542112, 0.0004293296951800585, 0.00137053313665092, 3.670702062663622e-05, 0.0015910869697108865, 0.00048218751908279955, 9.56441908783745e-06, 0.00015524028276558965, 0.00017542645218782127, 0.0004897234030067921, 6.953987031010911e-05, 0.0008855744963511825, 1.1304659892630298e-06, 4.837113374378532e-05, 0.0001840855402406305, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1554105579853058, 0.0020603404846042395, 5.904707904846873e-06, 0.005144172348082066, 0.00025823267060332, 0.0014501075493171811, 5.391828017309308e-05, 0.0021640402264893055, 2.1527968783630058e-05, 0.0016064351657405496, 0.010191709734499454, 0.010151085443794727, 0.7491754293441772, 0.0027421838603913784, 0.01017574779689312, 0.005842445883899927, 0.0030403982382267714, 3.083394767600112e-05, 0.006112964358180761, 0.016732651740312576, 0.00015879959391895682, 0.0020674674306064844, 0.0016374929109588265, 1.1047049156331923e-05, 0.0014025868149474263, 0.0001992537872865796, 0.001220017671585083, 2.1487703634193167e-05, 0.0007244196604005992, 5.2953896556573454e-06, 0.0011675384594127536, 0.0018881573341786861, 0.0071258689276874065, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.03857047110795975, 0.00048676287406124175, 8.812001084379517e-08, 0.000122779980301857, 0.005581730045378208, 0.0025055643636733294, 2.1938227291684598e-05, 0.00011670660751406103, 8.340171575582644e-08, 6.456951086875051e-05, 1.648088073125109e-05, 0.00033052131766453385, 0.0009664044482633471, 0.9410394430160522, 0.0025494787842035294, 0.000986628350801766, 0.00023048926959745586, 2.1702271624235436e-07, 9.28598819882609e-06, 0.001075997482985258, 8.328325202455744e-05, 0.0006203338853083551, 0.0003860929573420435, 1.7099183935442852e-07, 1.0056071005237754e-05, 0.0019989400170743465, 0.0020207997877150774, 4.052772510476643e-06, 1.9309491108288057e-05, 3.980539009518225e-09, 1.3065056009509135e-05, 4.567590394799481e-07, 5.0416052545187995e-05, 0.00011722340423148125, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.32133859395980835, 0.0015032502124086022, 0.00035717309219762683, 0.009410788305103779, 0.00016890434199012816, 0.016028327867388725, 0.0002053897042060271, 0.019911326467990875, 6.106824002927169e-05, 0.0025995566975325346, 0.0018900673603639007, 0.0003402142319828272, 0.018113745376467705, 0.0032540573738515377, 0.5562437772750854, 0.008188020437955856, 0.0034578992053866386, 2.759976268862374e-05, 0.00021329266019165516, 0.004111619666218758, 7.46081059332937e-05, 0.0024103508330881596, 0.0031445701606571674, 0.0002873913326766342, 0.0028005591593682766, 5.076818706584163e-05, 0.010558384470641613, 8.335074380738661e-05, 0.007083627860993147, 1.715210783004295e-05, 0.0007566043641418219, 0.00013426884834188968, 4.393113704281859e-05, 0.0036519141867756844, 0.0014778183540329337, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.16036191582679749, 0.0012780018150806427, 2.2185913621797226e-05, 0.0023713421542197466, 0.0002248762029921636, 0.001783479005098343, 0.003998146392405033, 0.00463333074003458, 4.07644038205035e-05, 0.0015416644746437669, 0.0015637290198355913, 0.00047258863924071193, 0.01683974452316761, 0.0034527594689279795, 0.017767908051609993, 0.7616898417472839, 0.004626939073204994, 0.00039177873986773193, 0.000968358654063195, 0.0050940741784870625, 3.4372260415693745e-06, 0.0016141458181664348, 0.0016999210929498076, 1.1049841305066366e-05, 0.00029084301786497235, 4.1652379877632484e-05, 0.000830529781524092, 0.00043556030141189694, 0.0004473649896681309, 1.989761585718952e-06, 0.0003503866319078952, 6.375126395141706e-05, 6.234755710465834e-05, 0.0025966004468500614, 0.0005663724150508642, 0.00186053360812366, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.16436104476451874, 0.00263788178563118, 4.14170281146653e-05, 0.06607695668935776, 0.0011643714969977736, 0.0015624640509486198, 0.01255388930439949, 0.0027841543778777122, 3.6274232115829363e-05, 0.004333572927862406, 0.006344497669488192, 0.00013702016440220177, 0.022220438346266747, 0.001139097730629146, 0.0015757762594148517, 0.003894534194841981, 0.5750948190689087, 0.0019101364305242896, 0.004216517321765423, 0.005103054456412792, 0.0011178136337548494, 0.011998940259218216, 0.006141388788819313, 8.207879727706313e-05, 0.0696118101477623, 0.0019772748928517103, 0.0024057570844888687, 0.01315633486956358, 0.0013538515195250511, 9.255067197955213e-06, 0.003937660250812769, 0.0014234132831916213, 5.669038000633009e-05, 0.005460998509079218, 0.0004025489615742117, 0.0006155924638733268, 0.003060702932998538, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.07928115874528885, 0.0006841905415058136, 6.815574948859648e-08, 0.0010951507138088346, 2.0406362182257e-06, 2.2378419089363888e-05, 0.0006314062047749758, 0.0003747308801393956, 2.3558166503789835e-05, 0.00018500992155168205, 0.008331519551575184, 7.647382153663784e-05, 0.00418297341093421, 2.444711526550236e-07, 1.1477874068077654e-05, 5.1371527661103755e-05, 0.0018863201839849353, 0.6889944076538086, 0.1946372091770172, 0.004766413476318121, 4.9866932386066765e-05, 0.0028069806285202503, 0.005558052100241184, 3.95493913174505e-07, 0.0008065987494774163, 3.353459078425658e-06, 5.2719624363817275e-05, 0.0003899818693753332, 0.0004888054681941867, 7.204272606031736e-06, 9.927168866852298e-05, 0.003168433904647827, 3.9500540879089385e-05, 0.0009883888997137547, 7.518784883586704e-08, 1.4315326097857906e-06, 6.484955520136282e-06, 0.00029431298025883734, 0.0, 0.0, 0.0, 0.0, 0.0], [0.28373828530311584, 0.004487877711653709, 5.058388069301145e-06, 0.019317252561450005, 5.4460702813230455e-05, 0.0012376051163300872, 0.0017604499589651823, 0.0063126953318715096, 0.00024137768195942044, 0.006789366248995066, 0.07160750776529312, 0.0003089959791395813, 0.06864359229803085, 4.332327443989925e-05, 0.0004192555497866124, 0.0020601244177669287, 0.006795847322791815, 0.01733478344976902, 0.33858922123908997, 0.03910181671380997, 0.0007660706178285182, 0.009993083775043488, 0.007829555310308933, 1.3115719411871396e-05, 0.01356492843478918, 6.270289304666221e-05, 0.0021037741098552942, 0.0009936662390828133, 0.00333457812666893, 9.826785390032455e-05, 0.006462769117206335, 0.03356613591313362, 0.00019831513054668903, 0.030609816312789917, 1.5713876564404927e-05, 0.0001204574218718335, 0.0005064337747171521, 0.001992761390283704, 0.018918951973319054, 0.0, 0.0, 0.0, 0.0], [0.25413063168525696, 0.007289606146514416, 1.6370242519769818e-05, 0.005644946824759245, 0.00047911194269545376, 0.003032122040167451, 0.0008039328386075795, 0.0005737085011787713, 0.001770934322848916, 0.0052279396913945675, 0.006573305930942297, 0.0037488588131964207, 0.039559945464134216, 0.0013069461565464735, 0.002049593720585108, 0.0032922588288784027, 0.005070534534752369, 0.0043929279781877995, 0.0029604455921798944, 0.5652295351028442, 0.006637280806899071, 0.020123612135648727, 0.013719636015594006, 3.509566158754751e-05, 0.00249256263487041, 0.0006804370786994696, 0.0050042420625686646, 0.0004963083192706108, 0.0002992783847730607, 0.0005683296476490796, 0.006156585179269314, 0.0014163292944431305, 0.002080241683870554, 0.01701081357896328, 0.0011587258195504546, 0.0008447651634924114, 0.0012247745180502534, 0.002057015197351575, 0.0027713908348232508, 0.0020689249504357576, 0.0, 0.0, 0.0], [0.02277885377407074, 0.00018681293295230716, 1.6159448890107342e-08, 0.0006753625930286944, 3.9720243876217864e-06, 1.8965230992762372e-05, 0.00014574530359823257, 0.00015363575948867947, 1.437089469646935e-08, 3.714960257639177e-06, 0.00038176553789526224, 9.189574484480545e-06, 0.00013344796025194228, 1.550033084640745e-05, 4.871678356721532e-06, 4.94858227284567e-07, 0.0004079081991221756, 1.881936623249203e-05, 3.495846613077447e-05, 0.00018379271205049008, 0.8935850262641907, 0.08001779019832611, 0.0004949963185936213, 6.95558668439844e-08, 0.0003611761494539678, 4.126275143789826e-06, 4.2135678086197004e-05, 0.0001535058399895206, 9.319555101683363e-05, 8.534826179129595e-10, 9.503900741947291e-07, 2.6511430405662395e-05, 9.136761605077481e-07, 4.42588907390018e-06, 3.6229419038136257e-06, 4.963176820638182e-07, 6.715119571509831e-09, 3.286409264546819e-05, 3.7280919968907256e-06, 3.7352272101998096e-06, 1.2850606253778096e-05, 0.0, 0.0], [0.04306096211075783, 0.00043120133341290057, 4.85736819655358e-08, 0.00029497192008420825, 6.446816769312136e-06, 1.794057789084036e-05, 5.2368359320098534e-05, 0.00012123115448048338, 8.686286356862638e-09, 4.085900854988722e-06, 0.00020099191169720143, 2.1525078409467824e-05, 0.0004682287108153105, 1.1544256267370656e-05, 3.74926021322608e-05, 7.720605935901403e-06, 0.0005387069541029632, 8.118282858049497e-05, 0.0002466407313477248, 0.0005759781342931092, 0.0023978566750884056, 0.9488213062286377, 0.002120599150657654, 3.640034407226267e-08, 7.682813156861812e-05, 2.893220880650915e-06, 1.444738245481858e-05, 1.2731605238514021e-05, 2.7991360184387304e-05, 3.7476780012646316e-10, 5.714333610740141e-07, 1.0850835678866133e-05, 3.7292174965841696e-06, 4.0900900785345584e-05, 3.826700321951648e-06, 6.126327207311988e-06, 5.169480346012278e-07, 4.086086119059473e-05, 1.2315524145378731e-05, 2.700609366002027e-05, 3.1881831091595814e-05, 0.00016747090558055788, 0.0], [0.26228708028793335, 0.04815318435430527, 0.0006689673755317926, 0.021233247593045235, 9.166418021777645e-05, 0.00039222234045155346, 0.00020396779291331768, 0.0021344097331166267, 0.001094849780201912, 0.004910758230835199, 0.006146768573671579, 0.0005123682785779238, 0.009844929911196232, 0.0001430303673259914, 0.0010762682650238276, 0.00034581031650304794, 0.010050452314317226, 0.008435730822384357, 0.008105281740427017, 0.00602466706186533, 0.0038135156501084566, 0.04830698296427727, 0.49913153052330017, 0.0012536782305687666, 0.033902086317539215, 0.00010517781629459932, 0.00037178199272602797, 8.200502634281293e-05, 0.0012060013832524419, 0.0001819609315134585, 0.0013505015522241592, 0.0009115547291003168, 0.00011081639240728691, 0.0016774724936112761, 3.677561107906513e-05, 0.00017677698633633554, 4.57889873359818e-05, 0.00101330759935081, 0.0026200786232948303, 0.002004811307415366, 0.0010430817492306232, 0.00036673821159638464, 0.008431942202150822]]], "attentionHeadNames": ["L6H9", "L5H5"], "tokens": ["<|endoftext|>", "Research", " in", " mechan", "istic", " interpret", "ability", " seeks", " to", " explain", " behaviors", " of", " machine", " learning", " models", " in", " terms", " of", " their", " internal", " components", ".", "Research", " in", " mechan", "istic", " interpret", "ability", " seeks", " to", " explain", " behaviors", " of", " machine", " learning", " models", " in", " terms", " of", " their", " internal", " components", "."]}
     )
@@ -2086,9 +2086,9 @@ <h4>Implications<a class="headerlink" href="#Implications" title="Permalink to t
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area rendered_html docutils container">
-<div>                            <div id="99df8c3d-54d9-4586-a67f-082a21e055a1" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("99df8c3d-54d9-4586-a67f-082a21e055a1")) {                    Plotly.newPlot(                        "99df8c3d-54d9-4586-a67f-082a21e055a1",                        [{"coloraxis":"coloraxis","name":"0","z":[[0.03906984627246857,0.0004489120910875499,0.03133600950241089,0.007519586011767387,0.034592170268297195,0.0003623008378781378,0.03451276570558548,0.19740214943885803,0.03844786807894707,0.04053793475031853,0.027628779411315918,0.02496313489973545],[0.18906497955322266,0.1721990704536438,0.06807752698659897,0.044945165514945984,0.0790855661034584,0.03096737712621689,0.028282105922698975,0.036443259567022324,0.026936713606119156,0.018826233223080635,0.04510088637471199,0.0065726605243980885],[0.15745531022548676,0.020724497735500336,0.48180025815963745,0.2991351783275604,0.1076490506529808,0.33004093170166016,0.09975510090589523,0.04926132783293724,0.2549382448196411,0.3606453239917755,0.12571798264980316,0.07931824773550034],[0.0058439951390028,0.1578730344772339,0.4189079701900482,0.30129143595695496,0.014345025643706322,0.03234415501356125,0.3312898576259613,0.5285982489585876,0.34241825342178345,0.10183659940958023,0.10516025871038437,0.22331231832504272],[0.10626537352800369,0.11930901557207108,0.02288069576025009,0.22826972603797913,0.02000402845442295,0.10010094195604324,0.17392224073410034,0.17407013475894928,0.025876855477690697,0.10249904543161392,0.009514803998172283,0.9921424388885498],[0.019766751676797867,0.005283255595713854,0.166484072804451,0.12087711691856384,0.1649997979402542,0.008032665587961674,0.4176996946334839,0.02582792565226555,0.04802580177783966,0.016231726855039597,0.031101685017347336,0.024261072278022766],[0.21729078888893127,0.03909975662827492,0.018048642203211784,0.059900619089603424,0.0329345278441906,0.0873454138636589,0.026895388960838318,0.09439495950937271,0.49926066398620605,0.00624012341722846,0.027026763185858727,0.1278562843799591],[0.25116443634033203,0.013308697380125523,0.006663368083536625,0.03743036091327667,0.02331540174782276,0.017407197505235672,0.02206745184957981,0.022141238674521446,0.04502354562282562,0.02084278129041195,0.008310752920806408,0.017167769372463226],[0.020890571177005768,0.016537854447960854,0.021583056077361107,0.015005743131041527,0.024211907759308815,0.1019899770617485,0.029100272804498672,0.22793740034103394,0.02781473658978939,0.01794101856648922,0.02482905425131321,0.03806224465370178],[0.02607565000653267,0.015407402068376541,0.020444145426154137,0.14558184146881104,0.01247023232281208,0.017151616513729095,0.013311783783137798,0.02445186860859394,0.018111703917384148,0.013193286024034023,0.03573964536190033,0.0187982190400362],[0.02147809788584709,0.018419133499264717,0.0181836299598217,0.021721404045820236,0.03156769275665283,0.03470622003078461,0.017550060525536537,0.011417530477046967,0.015795614570379257,0.04592300206422806,0.016215480864048004,0.030394626781344414],[0.033205095678567886,0.017571475356817245,0.015131049789488316,0.04148414358496666,0.015181177295744419,0.01758996769785881,0.01514849066734314,0.017676066607236862,0.06622707098722458,0.01845112442970276,0.01700744405388832,0.02974928542971611]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Previous Token Scores"}},                        {"responsive": true}                    ).then(function(){
+<div>                            <div id="a1e3ae9c-4880-482e-96c3-bfe0b0a07aa0" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("a1e3ae9c-4880-482e-96c3-bfe0b0a07aa0")) {                    Plotly.newPlot(                        "a1e3ae9c-4880-482e-96c3-bfe0b0a07aa0",                        [{"coloraxis":"coloraxis","name":"0","z":[[0.03906984627246857,0.0004489120910875499,0.03133600950241089,0.007519586011767387,0.034592170268297195,0.0003623008378781378,0.03451276570558548,0.19740214943885803,0.03844786807894707,0.04053793475031853,0.027628779411315918,0.02496313489973545],[0.18906497955322266,0.1721990704536438,0.06807752698659897,0.044945165514945984,0.0790855661034584,0.03096737712621689,0.028282105922698975,0.036443259567022324,0.026936713606119156,0.018826233223080635,0.04510088637471199,0.0065726605243980885],[0.15745531022548676,0.020724497735500336,0.48180025815963745,0.2991351783275604,0.1076490506529808,0.33004093170166016,0.09975510090589523,0.04926132783293724,0.2549382448196411,0.3606453239917755,0.12571798264980316,0.07931824773550034],[0.0058439951390028,0.1578730344772339,0.4189079701900482,0.30129143595695496,0.014345025643706322,0.03234415501356125,0.3312898576259613,0.5285982489585876,0.34241825342178345,0.10183659940958023,0.10516025871038437,0.22331231832504272],[0.10626537352800369,0.11930901557207108,0.02288069576025009,0.22826972603797913,0.02000402845442295,0.10010094195604324,0.17392224073410034,0.17407013475894928,0.025876855477690697,0.10249904543161392,0.009514803998172283,0.9921424388885498],[0.019766751676797867,0.005283255595713854,0.166484072804451,0.12087711691856384,0.1649997979402542,0.008032665587961674,0.4176996946334839,0.02582792565226555,0.04802580177783966,0.016231726855039597,0.031101685017347336,0.024261072278022766],[0.21729078888893127,0.03909975662827492,0.018048642203211784,0.059900619089603424,0.0329345278441906,0.0873454138636589,0.026895388960838318,0.09439495950937271,0.49926066398620605,0.00624012341722846,0.027026763185858727,0.1278562843799591],[0.25116443634033203,0.013308697380125523,0.006663368083536625,0.03743036091327667,0.02331540174782276,0.017407197505235672,0.02206745184957981,0.022141238674521446,0.04502354562282562,0.02084278129041195,0.008310752920806408,0.017167769372463226],[0.020890571177005768,0.016537854447960854,0.021583056077361107,0.015005743131041527,0.024211907759308815,0.1019899770617485,0.029100272804498672,0.22793740034103394,0.02781473658978939,0.01794101856648922,0.02482905425131321,0.03806224465370178],[0.02607565000653267,0.015407402068376541,0.020444145426154137,0.14558184146881104,0.01247023232281208,0.017151616513729095,0.013311783783137798,0.02445186860859394,0.018111703917384148,0.013193286024034023,0.03573964536190033,0.0187982190400362],[0.02147809788584709,0.018419133499264717,0.0181836299598217,0.021721404045820236,0.03156769275665283,0.03470622003078461,0.017550060525536537,0.011417530477046967,0.015795614570379257,0.04592300206422806,0.016215480864048004,0.030394626781344414],[0.033205095678567886,0.017571475356817245,0.015131049789488316,0.04148414358496666,0.015181177295744419,0.01758996769785881,0.01514849066734314,0.017676066607236862,0.06622707098722458,0.01845112442970276,0.01700744405388832,0.02974928542971611]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Previous Token Scores"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('99df8c3d-54d9-4586-a67f-082a21e055a1');
+var gd = document.getElementById('a1e3ae9c-4880-482e-96c3-bfe0b0a07aa0');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -2116,9 +2116,9 @@ <h4>Implications<a class="headerlink" href="#Implications" title="Permalink to t
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area rendered_html docutils container">
-<div>                            <div id="87727d60-5c53-428f-b583-3c0bee78d87a" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("87727d60-5c53-428f-b583-3c0bee78d87a")) {                    Plotly.newPlot(                        "87727d60-5c53-428f-b583-3c0bee78d87a",                        [{"coloraxis":"coloraxis","name":"0","z":[[0.003192325122654438,0.1323632299900055,0.005006915889680386,1.0427492270537186e-05,0.0013110269792377949,0.7034571170806885,0.004262051545083523,0.0001649669575272128,0.0024746304843574762,0.0008572910446673632,0.018891464918851852,0.008690940216183662],[0.0002916363882832229,0.00013781932648271322,0.0015036164550110698,0.005392488092184067,0.001858389819972217,0.009062976576387882,0.012414469383656979,0.00224055303260684,0.005135663319379091,0.005220625549554825,0.005546468310058117,0.029750702902674675],[0.0024816489312797785,0.009442185051739216,0.00034561342909000814,0.00025915156584233046,0.005211671814322472,0.0005709662800654769,0.0015209924895316362,0.006313098594546318,0.001560926903039217,0.00042158187716268003,0.0001535943301860243,0.005160389002412558],[0.6775667071342468,0.002840489149093628,0.0007841649930924177,0.004712563939392567,0.006322908215224743,0.00620671221986413,0.0005474375793710351,0.0003782894345931709,0.0020156530663371086,0.007952774874866009,0.003576836548745632,0.0026087891310453415],[0.008604029193520546,0.0070285736583173275,0.00759880430996418,0.0034427959471940994,0.016561321914196014,0.005979627370834351,0.0048697832971811295,0.0007624494028277695,0.006062139756977558,0.007536678109318018,0.012022883631289005,1.055450974014882e-12],[0.00950299110263586,0.008562120608985424,0.0041626025922596455,0.003008681582286954,0.006847443990409374,0.004358135629445314,0.007669272366911173,0.009584203362464905,0.007618824020028114,0.004328039940446615,0.04140293970704079,0.009761848486959934],[0.00445613032206893,0.008873283863067627,0.007405218668282032,0.006224926561117172,0.007319129537791014,0.005623883102089167,0.01734965480864048,0.005529423244297504,0.0029201731085777283,0.008636709302663803,0.006222232710570097,0.008358956314623356],[0.003699968568980694,0.041079357266426086,0.04148266464471817,0.009313643909990788,0.009097038768231869,0.008774377405643463,0.007298501208424568,0.023312222212553024,0.00884333811700344,0.00987985823303461,0.017598574981093407,0.006039811763912439],[0.008986336179077625,0.028667127713561058,0.00889119878411293,0.010114572942256927,0.009737403132021427,0.00761164166033268,0.009763195179402828,0.005155565217137337,0.009276354685425758,0.011895835399627686,0.010411957278847694,0.007498918566852808],[0.024409933015704155,0.011438388377428055,0.02003093995153904,0.005118591710925102,0.015081214718520641,0.012334463186562061,0.015452546067535877,0.008602438494563103,0.014702466316521168,0.020766181871294975,0.009192772209644318,0.005703327711671591],[0.017897022888064384,0.013280634768307209,0.0067551820538938046,0.012744802981615067,0.008020908571779728,0.007722198497503996,0.01734135113656521,0.0074547044932842255,0.007832478731870651,0.008252141997218132,0.013642732053995132,0.012807462364435196],[0.004923720378428698,0.007951012812554836,0.00794787798076868,0.004564068745821714,0.010363367386162281,0.009582011960446835,0.01028773095458746,0.008320694789290428,0.002570011653006077,0.012810957618057728,0.008063891902565956,0.0065582930110394955]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Duplicate Token Scores"}},                        {"responsive": true}                    ).then(function(){
+<div>                            <div id="d0ab7dd5-0224-4a12-bfb5-77c7a94ad500" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("d0ab7dd5-0224-4a12-bfb5-77c7a94ad500")) {                    Plotly.newPlot(                        "d0ab7dd5-0224-4a12-bfb5-77c7a94ad500",                        [{"coloraxis":"coloraxis","name":"0","z":[[0.003192325122654438,0.1323632299900055,0.005006915889680386,1.0427492270537186e-05,0.0013110269792377949,0.7034571170806885,0.004262051545083523,0.0001649669575272128,0.0024746304843574762,0.0008572910446673632,0.018891464918851852,0.008690940216183662],[0.0002916363882832229,0.00013781932648271322,0.0015036164550110698,0.005392488092184067,0.001858389819972217,0.009062976576387882,0.012414469383656979,0.00224055303260684,0.005135663319379091,0.005220625549554825,0.005546468310058117,0.029750702902674675],[0.0024816489312797785,0.009442185051739216,0.00034561342909000814,0.00025915156584233046,0.005211671814322472,0.0005709662800654769,0.0015209924895316362,0.006313098594546318,0.001560926903039217,0.00042158187716268003,0.0001535943301860243,0.005160389002412558],[0.6775667071342468,0.002840489149093628,0.0007841649930924177,0.004712563939392567,0.006322908215224743,0.00620671221986413,0.0005474375793710351,0.0003782894345931709,0.0020156530663371086,0.007952774874866009,0.003576836548745632,0.0026087891310453415],[0.008604029193520546,0.0070285736583173275,0.00759880430996418,0.0034427959471940994,0.016561321914196014,0.005979627370834351,0.0048697832971811295,0.0007624494028277695,0.006062139756977558,0.007536678109318018,0.012022883631289005,1.055450974014882e-12],[0.00950299110263586,0.008562120608985424,0.0041626025922596455,0.003008681582286954,0.006847443990409374,0.004358135629445314,0.007669272366911173,0.009584203362464905,0.007618824020028114,0.004328039940446615,0.04140293970704079,0.009761848486959934],[0.00445613032206893,0.008873283863067627,0.007405218668282032,0.006224926561117172,0.007319129537791014,0.005623883102089167,0.01734965480864048,0.005529423244297504,0.0029201731085777283,0.008636709302663803,0.006222232710570097,0.008358956314623356],[0.003699968568980694,0.041079357266426086,0.04148266464471817,0.009313643909990788,0.009097038768231869,0.008774377405643463,0.007298501208424568,0.023312222212553024,0.00884333811700344,0.00987985823303461,0.017598574981093407,0.006039811763912439],[0.008986336179077625,0.028667127713561058,0.00889119878411293,0.010114572942256927,0.009737403132021427,0.00761164166033268,0.009763195179402828,0.005155565217137337,0.009276354685425758,0.011895835399627686,0.010411957278847694,0.007498918566852808],[0.024409933015704155,0.011438388377428055,0.02003093995153904,0.005118591710925102,0.015081214718520641,0.012334463186562061,0.015452546067535877,0.008602438494563103,0.014702466316521168,0.020766181871294975,0.009192772209644318,0.005703327711671591],[0.017897022888064384,0.013280634768307209,0.0067551820538938046,0.012744802981615067,0.008020908571779728,0.007722198497503996,0.01734135113656521,0.0074547044932842255,0.007832478731870651,0.008252141997218132,0.013642732053995132,0.012807462364435196],[0.004923720378428698,0.007951012812554836,0.00794787798076868,0.004564068745821714,0.010363367386162281,0.009582011960446835,0.01028773095458746,0.008320694789290428,0.002570011653006077,0.012810957618057728,0.008063891902565956,0.0065582930110394955]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Duplicate Token Scores"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('87727d60-5c53-428f-b583-3c0bee78d87a');
+var gd = document.getElementById('d0ab7dd5-0224-4a12-bfb5-77c7a94ad500');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -2146,9 +2146,9 @@ <h4>Implications<a class="headerlink" href="#Implications" title="Permalink to t
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area rendered_html docutils container">
-<div>                            <div id="c40872d7-beb3-47ab-b81d-dccc77f7f064" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("c40872d7-beb3-47ab-b81d-dccc77f7f064")) {                    Plotly.newPlot(                        "c40872d7-beb3-47ab-b81d-dccc77f7f064",                        [{"coloraxis":"coloraxis","name":"0","z":[[0.004035576246678829,3.85936327802483e-05,0.003946060314774513,1.7428237697458826e-07,5.9896454331465065e-05,4.0836926928022876e-05,0.003501761006191373,0.00024610935361124575,0.003167978022247553,0.003010402200743556,0.0020935393404215574,0.008525436744093895],[0.0005264790961518884,0.0001567041763337329,0.0015079419827088714,0.005595334805548191,0.0018401179695501924,0.003887568600475788,0.005349150858819485,0.0046491725370287895,0.005880181211978197,0.007283915765583515,0.005552181042730808,0.00012677432096097618],[0.0022015569265931845,0.008784865029156208,0.0021590511314570904,0.0010448142420500517,0.005142332520335913,0.002251654164865613,0.0008376692421734333,0.0063524022698402405,0.00261817779392004,0.0010309149511158466,0.00015219332999549806,0.005351166240870953],[0.007752286270260811,0.003091607242822647,0.00136294006370008,0.004341903142631054,0.011233086697757244,0.006535585504025221,0.0009068045765161514,0.0006078635924495757,0.0028195830527693033,0.005254121031612158,0.004195701330900192,0.0025541584473103285],[0.007342719938606024,0.004788315389305353,0.007458813022822142,0.0033073171507567167,0.007871860638260841,0.004219716414809227,0.004172018263489008,0.0005154716782271862,0.008124986663460732,0.006826961878687143,0.008085506968200207,3.761463363005646e-11],[0.43377670645713806,0.9306100010871887,0.006382290739566088,0.003473056945949793,0.005501003935933113,0.9255975484848022,0.005381471943110228,0.007857300341129303,0.008637800812721252,0.015764445066452026,0.012188390828669071,0.008265750482678413],[0.0025073012802749872,0.008432051166892052,0.008623287081718445,0.007653359789401293,0.011058016680181026,0.005525414831936359,0.017205143347382545,0.004794336389750242,0.004097685217857361,0.9257786273956299,0.020375533029437065,0.006313955411314964],[0.005555589683353901,0.18942901492118835,0.8509916663169861,0.008273174054920673,0.008239682763814926,0.00864996574819088,0.028328388929367065,0.08996234834194183,0.0066174231469631195,0.009413909167051315,0.9037811756134033,0.03037119098007679],[0.0073545570485293865,0.3791305124759674,0.005602711346000433,0.025401653721928596,0.008504705503582954,0.00623104115948081,0.1189238503575325,0.0051146638579666615,0.013350976631045341,0.01576736569404602,0.025844166055321693,0.008429795503616333],[0.23989056050777435,0.14378714561462402,0.09330623596906662,0.005819808691740036,0.07744759321212769,0.01644795574247837,0.4442824423313141,0.011141379363834858,0.03618974611163139,0.47264590859413147,0.008039995096623898,0.030952973291277885],[0.36065515875816345,0.4820112884044647,0.02285100519657135,0.12641923129558563,0.04125585779547691,0.007237402722239494,0.287715345621109,0.3897298574447632,0.0300607867538929,0.006112930364906788,0.16554805636405945,0.2224510908126831],[0.007408502046018839,0.033737070858478546,0.020412690937519073,0.0027554105035960674,0.02518630214035511,0.07808848470449448,0.033082809299230576,0.046439945697784424,0.0032543179113417864,0.2744251787662506,0.3800223767757416,0.009483428671956062]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Induction Head Scores"}},                        {"responsive": true}                    ).then(function(){
+<div>                            <div id="db9eb568-d240-4b5e-ada7-72a996ee0efc" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("db9eb568-d240-4b5e-ada7-72a996ee0efc")) {                    Plotly.newPlot(                        "db9eb568-d240-4b5e-ada7-72a996ee0efc",                        [{"coloraxis":"coloraxis","name":"0","z":[[0.004035576246678829,3.85936327802483e-05,0.003946060314774513,1.7428237697458826e-07,5.9896454331465065e-05,4.0836926928022876e-05,0.003501761006191373,0.00024610935361124575,0.003167978022247553,0.003010402200743556,0.0020935393404215574,0.008525436744093895],[0.0005264790961518884,0.0001567041763337329,0.0015079419827088714,0.005595334805548191,0.0018401179695501924,0.003887568600475788,0.005349150858819485,0.0046491725370287895,0.005880181211978197,0.007283915765583515,0.005552181042730808,0.00012677432096097618],[0.0022015569265931845,0.008784865029156208,0.0021590511314570904,0.0010448142420500517,0.005142332520335913,0.002251654164865613,0.0008376692421734333,0.0063524022698402405,0.00261817779392004,0.0010309149511158466,0.00015219332999549806,0.005351166240870953],[0.007752286270260811,0.003091607242822647,0.00136294006370008,0.004341903142631054,0.011233086697757244,0.006535585504025221,0.0009068045765161514,0.0006078635924495757,0.0028195830527693033,0.005254121031612158,0.004195701330900192,0.0025541584473103285],[0.007342719938606024,0.004788315389305353,0.007458813022822142,0.0033073171507567167,0.007871860638260841,0.004219716414809227,0.004172018263489008,0.0005154716782271862,0.008124986663460732,0.006826961878687143,0.008085506968200207,3.761463363005646e-11],[0.43377670645713806,0.9306100010871887,0.006382290739566088,0.003473056945949793,0.005501003935933113,0.9255975484848022,0.005381471943110228,0.007857300341129303,0.008637800812721252,0.015764445066452026,0.012188390828669071,0.008265750482678413],[0.0025073012802749872,0.008432051166892052,0.008623287081718445,0.007653359789401293,0.011058016680181026,0.005525414831936359,0.017205143347382545,0.004794336389750242,0.004097685217857361,0.9257786273956299,0.020375533029437065,0.006313955411314964],[0.005555589683353901,0.18942901492118835,0.8509916663169861,0.008273174054920673,0.008239682763814926,0.00864996574819088,0.028328388929367065,0.08996234834194183,0.0066174231469631195,0.009413909167051315,0.9037811756134033,0.03037119098007679],[0.0073545570485293865,0.3791305124759674,0.005602711346000433,0.025401653721928596,0.008504705503582954,0.00623104115948081,0.1189238503575325,0.0051146638579666615,0.013350976631045341,0.01576736569404602,0.025844166055321693,0.008429795503616333],[0.23989056050777435,0.14378714561462402,0.09330623596906662,0.005819808691740036,0.07744759321212769,0.01644795574247837,0.4442824423313141,0.011141379363834858,0.03618974611163139,0.47264590859413147,0.008039995096623898,0.030952973291277885],[0.36065515875816345,0.4820112884044647,0.02285100519657135,0.12641923129558563,0.04125585779547691,0.007237402722239494,0.287715345621109,0.3897298574447632,0.0300607867538929,0.006112930364906788,0.16554805636405945,0.2224510908126831],[0.007408502046018839,0.033737070858478546,0.020412690937519073,0.0027554105035960674,0.02518630214035511,0.07808848470449448,0.033082809299230576,0.046439945697784424,0.0032543179113417864,0.2744251787662506,0.3800223767757416,0.009483428671956062]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Induction Head Scores"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('c40872d7-beb3-47ab-b81d-dccc77f7f064');
+var gd = document.getElementById('db9eb568-d240-4b5e-ada7-72a996ee0efc');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -2278,9 +2278,9 @@ <h3>Backup Name Mover Heads<a class="headerlink" href="#Backup-Name-Mover-Heads"
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area rendered_html docutils container">
-<div>                            <div id="63e4aa76-ac0f-4935-b710-49e27e2d83fb" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("63e4aa76-ac0f-4935-b710-49e27e2d83fb")) {                    Plotly.newPlot(                        "63e4aa76-ac0f-4935-b710-49e27e2d83fb",                        [{"coloraxis":"coloraxis","name":"0","z":[[-0.002156492555513978,-0.00046507816296070814,0.00024167407536879182,0.00028066313825547695,-0.00041629592305980623,-0.0004892397555522621,-0.0026209561619907618,-0.0029356726445257664,0.000425610167440027,0.0005418087239377201,0.0002375440817559138,-7.489288691431284e-05],[-0.0006585267838090658,0.00040605987305752933,-0.0009330196771770716,0.0008937990060076118,-0.0009785185102373362,-0.0005338399787433445,-0.0027988061774522066,-0.004214102402329445,0.0025785816833376884,0.002450671512633562,0.0005351606523618102,0.0012349870521575212],[0.0009405002347193658,-0.0011168736964464188,-0.0011542305583134294,-0.0015697323251515627,-0.0005699749453924596,0.0014514662325382233,0.0024399266112595797,0.0031583067029714584,0.0009236747864633799,-0.00357811083085835,-0.0010650111362338066,-0.000355880125425756],[-0.0005624890327453613,-1.1902564438059926e-05,0.0011530898045748472,0.0007360299350693822,0.0016493811272084713,0.0008800593786872923,-0.0006905524642206728,-0.0030319970101118088,0.0008080819388851523,0.00010364333866164088,-0.0005807871930301189,-0.0011067159939557314],[-0.0026376438327133656,0.00026929602609016,-0.001641781534999609,-0.0034070422407239676,0.001744971377775073,0.00046448659850284457,-0.000789930927567184,0.0018328832229599357,-0.0008632286335341632,-0.00039789103902876377,0.000787912926170975,-0.00012039250577799976],[0.0008688485249876976,0.0009473453392274678,-0.0022809950169175863,-0.0011803111992776394,0.00024082366144284606,-0.0004318496794439852,-0.0003728233277797699,-0.0007385354256257415,0.0008114463416859508,-0.00040441250894218683,-0.007074240129441023,0.003946419805288315],[-0.01491759717464447,-0.002280194777995348,0.0022679578978568316,-8.297746535390615e-05,-0.004981024190783501,0.0027670864947140217,0.00626616645604372,-0.0034861797466874123,-0.0013347704662010074,-0.0017918730154633522,-0.001223177881911397,0.000405364902690053],[-0.00024617952294647694,-0.0057901544496417046,-0.0004975934862159193,0.14218278229236603,-0.001496216980740428,-0.01900637149810791,0.0031333654187619686,-0.0018582025077193975,-0.011305071413516998,0.19225990772247314,-0.0011892484035342932,-0.0010282367002218962],[-0.0038004019297659397,-0.0008571128128096461,-0.013956686481833458,0.008289236575365067,0.004314992111176252,-0.009073692373931408,-0.08315175771713257,0.0034568109549582005,-0.018055014312267303,0.00217800703831017,0.29780468344688416,0.02409377135336399],[0.08904657512903214,-0.0007931506261229515,0.07247606664896011,0.015016509220004082,-0.021209245547652245,0.052054572850465775,1.441115140914917,0.04743725806474686,-0.0322909839451313,0.0,0.0019993034657090902,-0.008072325959801674],[0.8600812554359436,0.3260071277618408,0.16344299912452698,0.07133564352989197,-0.004448350518941879,0.0006813818472437561,0.36613476276397705,-0.710505485534668,-0.0020313779823482037,-0.032143473625183105,1.2294319868087769,0.0018451482756063342],[0.016877301037311554,-0.001730009913444519,-0.501085638999939,0.02749774605035782,-0.005966278724372387,-0.0049441163428127766,-0.08855222165584564,0.006622320972383022,0.04412432014942169,-0.027266837656497955,-1.1349124908447266,0.02287965640425682]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"margin":{"t":60}},                        {"responsive": true}                    ).then(function(){
+<div>                            <div id="17e8a56b-4819-438b-b954-b94f0a90174a" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("17e8a56b-4819-438b-b954-b94f0a90174a")) {                    Plotly.newPlot(                        "17e8a56b-4819-438b-b954-b94f0a90174a",                        [{"coloraxis":"coloraxis","name":"0","z":[[-0.002156492555513978,-0.00046507816296070814,0.00024167407536879182,0.00028066313825547695,-0.00041629592305980623,-0.0004892397555522621,-0.0026209561619907618,-0.0029356726445257664,0.000425610167440027,0.0005418087239377201,0.0002375440817559138,-7.489288691431284e-05],[-0.0006585267838090658,0.00040605987305752933,-0.0009330196771770716,0.0008937990060076118,-0.0009785185102373362,-0.0005338399787433445,-0.0027988061774522066,-0.004214102402329445,0.0025785816833376884,0.002450671512633562,0.0005351606523618102,0.0012349870521575212],[0.0009405002347193658,-0.0011168736964464188,-0.0011542305583134294,-0.0015697323251515627,-0.0005699749453924596,0.0014514662325382233,0.0024399266112595797,0.0031583067029714584,0.0009236747864633799,-0.00357811083085835,-0.0010650111362338066,-0.000355880125425756],[-0.0005624890327453613,-1.1902564438059926e-05,0.0011530898045748472,0.0007360299350693822,0.0016493811272084713,0.0008800593786872923,-0.0006905524642206728,-0.0030319970101118088,0.0008080819388851523,0.00010364333866164088,-0.0005807871930301189,-0.0011067159939557314],[-0.0026376438327133656,0.00026929602609016,-0.001641781534999609,-0.0034070422407239676,0.001744971377775073,0.00046448659850284457,-0.000789930927567184,0.0018328832229599357,-0.0008632286335341632,-0.00039789103902876377,0.000787912926170975,-0.00012039250577799976],[0.0008688485249876976,0.0009473453392274678,-0.0022809950169175863,-0.0011803111992776394,0.00024082366144284606,-0.0004318496794439852,-0.0003728233277797699,-0.0007385354256257415,0.0008114463416859508,-0.00040441250894218683,-0.007074240129441023,0.003946419805288315],[-0.01491759717464447,-0.002280194777995348,0.0022679578978568316,-8.297746535390615e-05,-0.004981024190783501,0.0027670864947140217,0.00626616645604372,-0.0034861797466874123,-0.0013347704662010074,-0.0017918730154633522,-0.001223177881911397,0.000405364902690053],[-0.00024617952294647694,-0.0057901544496417046,-0.0004975934862159193,0.14218278229236603,-0.001496216980740428,-0.01900637149810791,0.0031333654187619686,-0.0018582025077193975,-0.011305071413516998,0.19225990772247314,-0.0011892484035342932,-0.0010282367002218962],[-0.0038004019297659397,-0.0008571128128096461,-0.013956686481833458,0.008289236575365067,0.004314992111176252,-0.009073692373931408,-0.08315175771713257,0.0034568109549582005,-0.018055014312267303,0.00217800703831017,0.29780468344688416,0.02409377135336399],[0.08904657512903214,-0.0007931506261229515,0.07247606664896011,0.015016509220004082,-0.021209245547652245,0.052054572850465775,1.441115140914917,0.04743725806474686,-0.0322909839451313,0.0,0.0019993034657090902,-0.008072325959801674],[0.8600812554359436,0.3260071277618408,0.16344299912452698,0.07133564352989197,-0.004448350518941879,0.0006813818472437561,0.36613476276397705,-0.710505485534668,-0.0020313779823482037,-0.032143473625183105,1.2294319868087769,0.0018451482756063342],[0.016877301037311554,-0.001730009913444519,-0.501085638999939,0.02749774605035782,-0.005966278724372387,-0.0049441163428127766,-0.08855222165584564,0.006622320972383022,0.04412432014942169,-0.027266837656497955,-1.1349124908447266,0.02287965640425682]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"margin":{"t":60}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('63e4aa76-ac0f-4935-b710-49e27e2d83fb');
+var gd = document.getElementById('17e8a56b-4819-438b-b954-b94f0a90174a');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -2308,9 +2308,9 @@ <h3>Backup Name Mover Heads<a class="headerlink" href="#Backup-Name-Mover-Heads"
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area rendered_html docutils container">
-<div>                            <div id="aed2d9cd-0ab8-4309-b50e-32156504e9cc" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("aed2d9cd-0ab8-4309-b50e-32156504e9cc")) {                    Plotly.newPlot(                        "aed2d9cd-0ab8-4309-b50e-32156504e9cc",                        [{"hovertemplate":"\u003cb\u003e%{hovertext}\u003c\u002fb\u003e\u003cbr\u003e\u003cbr\u003eAblated=%{x}\u003cbr\u003eOriginal=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","hovertext":["L0H0","L0H1","L0H2","L0H3","L0H4","L0H5","L0H6","L0H7","L0H8","L0H9","L0H10","L0H11","L1H0","L1H1","L1H2","L1H3","L1H4","L1H5","L1H6","L1H7","L1H8","L1H9","L1H10","L1H11","L2H0","L2H1","L2H2","L2H3","L2H4","L2H5","L2H6","L2H7","L2H8","L2H9","L2H10","L2H11","L3H0","L3H1","L3H2","L3H3","L3H4","L3H5","L3H6","L3H7","L3H8","L3H9","L3H10","L3H11","L4H0","L4H1","L4H2","L4H3","L4H4","L4H5","L4H6","L4H7","L4H8","L4H9","L4H10","L4H11","L5H0","L5H1","L5H2","L5H3","L5H4","L5H5","L5H6","L5H7","L5H8","L5H9","L5H10","L5H11","L6H0","L6H1","L6H2","L6H3","L6H4","L6H5","L6H6","L6H7","L6H8","L6H9","L6H10","L6H11","L7H0","L7H1","L7H2","L7H3","L7H4","L7H5","L7H6","L7H7","L7H8","L7H9","L7H10","L7H11","L8H0","L8H1","L8H2","L8H3","L8H4","L8H5","L8H6","L8H7","L8H8","L8H9","L8H10","L8H11","L9H0","L9H1","L9H2","L9H3","L9H4","L9H5","L9H6","L9H7","L9H8","L9H9","L9H10","L9H11","L10H0","L10H1","L10H2","L10H3","L10H4","L10H5","L10H6","L10H7","L10H8","L10H9","L10H10","L10H11","L11H0","L11H1","L11H2","L11H3","L11H4","L11H5","L11H6","L11H7","L11H8","L11H9","L11H10","L11H11"],"legendgroup":"","marker":{"color":"#636efa","symbol":"circle"},"mode":"markers","name":"","orientation":"v","showlegend":false,"x":[-0.002156492555513978,-0.00046507816296070814,0.00024167407536879182,0.00028066313825547695,-0.00041629592305980623,-0.0004892397555522621,-0.0026209561619907618,-0.0029356726445257664,0.000425610167440027,0.0005418087239377201,0.0002375440817559138,-7.489288691431284e-05,-0.0006585267838090658,0.00040605987305752933,-0.0009330196771770716,0.0008937990060076118,-0.0009785185102373362,-0.0005338399787433445,-0.0027988061774522066,-0.004214102402329445,0.0025785816833376884,0.002450671512633562,0.0005351606523618102,0.0012349870521575212,0.0009405002347193658,-0.0011168736964464188,-0.0011542305583134294,-0.0015697323251515627,-0.0005699749453924596,0.0014514662325382233,0.0024399266112595797,0.0031583067029714584,0.0009236747864633799,-0.00357811083085835,-0.0010650111362338066,-0.000355880125425756,-0.0005624890327453613,-1.1902564438059926e-05,0.0011530898045748472,0.0007360299350693822,0.0016493811272084713,0.0008800593786872923,-0.0006905524642206728,-0.0030319970101118088,0.0008080819388851523,0.00010364333866164088,-0.0005807871930301189,-0.0011067159939557314,-0.0026376438327133656,0.00026929602609016,-0.001641781534999609,-0.0034070422407239676,0.001744971377775073,0.00046448659850284457,-0.000789930927567184,0.0018328832229599357,-0.0008632286335341632,-0.00039789103902876377,0.000787912926170975,-0.00012039250577799976,0.0008688485249876976,0.0009473453392274678,-0.0022809950169175863,-0.0011803111992776394,0.00024082366144284606,-0.0004318496794439852,-0.0003728233277797699,-0.0007385354256257415,0.0008114463416859508,-0.00040441250894218683,-0.007074240129441023,0.003946419805288315,-0.01491759717464447,-0.002280194777995348,0.0022679578978568316,-8.297746535390615e-05,-0.004981024190783501,0.0027670864947140217,0.00626616645604372,-0.0034861797466874123,-0.0013347704662010074,-0.0017918730154633522,-0.001223177881911397,0.000405364902690053,-0.00024617952294647694,-0.0057901544496417046,-0.0004975934862159193,0.14218278229236603,-0.001496216980740428,-0.01900637149810791,0.0031333654187619686,-0.0018582025077193975,-0.011305071413516998,0.19225990772247314,-0.0011892484035342932,-0.0010282367002218962,-0.0038004019297659397,-0.0008571128128096461,-0.013956686481833458,0.008289236575365067,0.004314992111176252,-0.009073692373931408,-0.08315175771713257,0.0034568109549582005,-0.018055014312267303,0.00217800703831017,0.29780468344688416,0.02409377135336399,0.08904657512903214,-0.0007931506261229515,0.07247606664896011,0.015016509220004082,-0.021209245547652245,0.052054572850465775,1.441115140914917,0.04743725806474686,-0.0322909839451313,0.0,0.0019993034657090902,-0.008072325959801674,0.8600812554359436,0.3260071277618408,0.16344299912452698,0.07133564352989197,-0.004448350518941879,0.0006813818472437561,0.36613476276397705,-0.710505485534668,-0.0020313779823482037,-0.032143473625183105,1.2294319868087769,0.0018451482756063342,0.016877301037311554,-0.001730009913444519,-0.501085638999939,0.02749774605035782,-0.005966278724372387,-0.0049441163428127766,-0.08855222165584564,0.006622320972383022,0.04412432014942169,-0.027266837656497955,-1.1349124908447266,0.02287965640425682],"xaxis":"x","y":[-0.0020563285797834396,-0.0005101955030113459,0.0004685768508352339,0.00012511832755990326,-0.0006028721109032631,-0.00024295502225868404,-0.002318894723430276,-0.002758359769359231,0.0005645868368446827,0.000969740329310298,-0.0002504501899238676,4.740082658827305e-06,-0.0010071131400763988,0.0003947088844142854,-0.0015487205237150192,0.0014034901978448033,-0.0012652688892558217,-0.0011358250631019473,-0.0028159404173493385,-0.0029645359609276056,0.0029190238565206528,0.0025743518490344286,0.00036237656604498625,0.0017548884497955441,0.0005569332861341536,-0.00112663593608886,-0.0017354178708046675,-0.0014514722861349583,-0.00028740704874508083,0.0017210595542564988,0.00266590085811913,0.0031146793626248837,0.0005667305667884648,-0.0036664949730038643,-0.0018847067840397358,7.027178071439266e-06,-0.000726439815480262,0.00011369686399120837,0.001430142787285149,0.000749052269384265,0.0020184761378914118,0.0007436758605763316,-0.0004617759259417653,-0.003905785735696554,0.001140733016654849,-4.027335671707988e-05,-0.0013293256051838398,-0.0017636881675571203,-0.002828173339366913,0.0003364472358953208,-0.0014249038649722934,-0.0037773081567138433,0.0015998876187950373,0.00029889732832089067,-0.0008046309230849147,0.0020388164557516575,-0.0015593776479363441,-0.0006437147385440767,0.001116806990467012,-0.00035003889934159815,0.0011338151525706053,0.0011259106686338782,-0.0025163597892969847,-0.0014790240675210953,0.00038791983388364315,-6.407807813957334e-05,-0.0005096746608614922,-0.0008841876406222582,0.0006399309495463967,-0.001009696745313704,-0.0067590330727398396,0.0033667273819446564,-0.015147387981414795,-0.002135086804628372,0.002593189012259245,-0.00042674108408391476,-0.005559002980589867,0.0026659294962882996,0.006410874892026186,-0.0038270100485533476,-0.00038422830402851105,-0.0016430210089311004,-0.0013344308827072382,-9.184109512716532e-05,-9.488123760093004e-05,-0.005788922309875488,-0.0006383719155564904,0.134933739900589,-0.0017687628278508782,-0.018917974084615707,0.0038733629044145346,-0.002145076170563698,-0.010327237658202648,0.18325874209403992,-0.0007747883792035282,-0.0010452116839587688,-0.003833947703242302,-0.000804627372417599,-0.012673338875174522,0.008045812137424946,0.0036040153354406357,-0.009398169815540314,-0.08272106945514679,0.0035550352185964584,-0.018404126167297363,0.0017586719477549195,0.2896132469177246,0.022854045033454895,0.08595201373100281,-0.0006932567339390516,0.06816966831684113,0.01311141811311245,-0.021098004654049873,0.05112440511584282,1.384489893913269,0.04583733528852463,-0.038303446024656296,2.9854445457458496,0.001966139767318964,-0.008030213415622711,0.560872495174408,0.17083144187927246,-0.033618733286857605,0.05821547657251358,-0.0024530075024813414,0.0018771879840642214,0.2882729768753052,-1.898641586303711,-0.001528693363070488,-0.03513003885746002,0.48021769523620605,-0.0009116916917264462,0.0160758625715971,-0.03986112400889397,-0.3879111707210541,0.011123226955533028,-0.005477802362293005,-0.0025129495188593864,-0.08056114614009857,0.007518642581999302,0.043011054396629333,-0.0400824099779129,-0.9702335596084595,0.011862391605973244],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"Ablated"},"range":[-3,3]},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"Original"},"range":[-3,3]},"legend":{"tracegroupgap":0},"title":{"text":"Original vs Post-Ablation Direct Logit Attribution of Heads"}},                        {"responsive": true}                    ).then(function(){
+<div>                            <div id="6e7a7d6f-0a22-4385-834f-25d48524e13a" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                require(["plotly"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("6e7a7d6f-0a22-4385-834f-25d48524e13a")) {                    Plotly.newPlot(                        "6e7a7d6f-0a22-4385-834f-25d48524e13a",                        [{"hovertemplate":"\u003cb\u003e%{hovertext}\u003c\u002fb\u003e\u003cbr\u003e\u003cbr\u003eAblated=%{x}\u003cbr\u003eOriginal=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","hovertext":["L0H0","L0H1","L0H2","L0H3","L0H4","L0H5","L0H6","L0H7","L0H8","L0H9","L0H10","L0H11","L1H0","L1H1","L1H2","L1H3","L1H4","L1H5","L1H6","L1H7","L1H8","L1H9","L1H10","L1H11","L2H0","L2H1","L2H2","L2H3","L2H4","L2H5","L2H6","L2H7","L2H8","L2H9","L2H10","L2H11","L3H0","L3H1","L3H2","L3H3","L3H4","L3H5","L3H6","L3H7","L3H8","L3H9","L3H10","L3H11","L4H0","L4H1","L4H2","L4H3","L4H4","L4H5","L4H6","L4H7","L4H8","L4H9","L4H10","L4H11","L5H0","L5H1","L5H2","L5H3","L5H4","L5H5","L5H6","L5H7","L5H8","L5H9","L5H10","L5H11","L6H0","L6H1","L6H2","L6H3","L6H4","L6H5","L6H6","L6H7","L6H8","L6H9","L6H10","L6H11","L7H0","L7H1","L7H2","L7H3","L7H4","L7H5","L7H6","L7H7","L7H8","L7H9","L7H10","L7H11","L8H0","L8H1","L8H2","L8H3","L8H4","L8H5","L8H6","L8H7","L8H8","L8H9","L8H10","L8H11","L9H0","L9H1","L9H2","L9H3","L9H4","L9H5","L9H6","L9H7","L9H8","L9H9","L9H10","L9H11","L10H0","L10H1","L10H2","L10H3","L10H4","L10H5","L10H6","L10H7","L10H8","L10H9","L10H10","L10H11","L11H0","L11H1","L11H2","L11H3","L11H4","L11H5","L11H6","L11H7","L11H8","L11H9","L11H10","L11H11"],"legendgroup":"","marker":{"color":"#636efa","symbol":"circle"},"mode":"markers","name":"","orientation":"v","showlegend":false,"x":[-0.002156492555513978,-0.00046507816296070814,0.00024167407536879182,0.00028066313825547695,-0.00041629592305980623,-0.0004892397555522621,-0.0026209561619907618,-0.0029356726445257664,0.000425610167440027,0.0005418087239377201,0.0002375440817559138,-7.489288691431284e-05,-0.0006585267838090658,0.00040605987305752933,-0.0009330196771770716,0.0008937990060076118,-0.0009785185102373362,-0.0005338399787433445,-0.0027988061774522066,-0.004214102402329445,0.0025785816833376884,0.002450671512633562,0.0005351606523618102,0.0012349870521575212,0.0009405002347193658,-0.0011168736964464188,-0.0011542305583134294,-0.0015697323251515627,-0.0005699749453924596,0.0014514662325382233,0.0024399266112595797,0.0031583067029714584,0.0009236747864633799,-0.00357811083085835,-0.0010650111362338066,-0.000355880125425756,-0.0005624890327453613,-1.1902564438059926e-05,0.0011530898045748472,0.0007360299350693822,0.0016493811272084713,0.0008800593786872923,-0.0006905524642206728,-0.0030319970101118088,0.0008080819388851523,0.00010364333866164088,-0.0005807871930301189,-0.0011067159939557314,-0.0026376438327133656,0.00026929602609016,-0.001641781534999609,-0.0034070422407239676,0.001744971377775073,0.00046448659850284457,-0.000789930927567184,0.0018328832229599357,-0.0008632286335341632,-0.00039789103902876377,0.000787912926170975,-0.00012039250577799976,0.0008688485249876976,0.0009473453392274678,-0.0022809950169175863,-0.0011803111992776394,0.00024082366144284606,-0.0004318496794439852,-0.0003728233277797699,-0.0007385354256257415,0.0008114463416859508,-0.00040441250894218683,-0.007074240129441023,0.003946419805288315,-0.01491759717464447,-0.002280194777995348,0.0022679578978568316,-8.297746535390615e-05,-0.004981024190783501,0.0027670864947140217,0.00626616645604372,-0.0034861797466874123,-0.0013347704662010074,-0.0017918730154633522,-0.001223177881911397,0.000405364902690053,-0.00024617952294647694,-0.0057901544496417046,-0.0004975934862159193,0.14218278229236603,-0.001496216980740428,-0.01900637149810791,0.0031333654187619686,-0.0018582025077193975,-0.011305071413516998,0.19225990772247314,-0.0011892484035342932,-0.0010282367002218962,-0.0038004019297659397,-0.0008571128128096461,-0.013956686481833458,0.008289236575365067,0.004314992111176252,-0.009073692373931408,-0.08315175771713257,0.0034568109549582005,-0.018055014312267303,0.00217800703831017,0.29780468344688416,0.02409377135336399,0.08904657512903214,-0.0007931506261229515,0.07247606664896011,0.015016509220004082,-0.021209245547652245,0.052054572850465775,1.441115140914917,0.04743725806474686,-0.0322909839451313,0.0,0.0019993034657090902,-0.008072325959801674,0.8600812554359436,0.3260071277618408,0.16344299912452698,0.07133564352989197,-0.004448350518941879,0.0006813818472437561,0.36613476276397705,-0.710505485534668,-0.0020313779823482037,-0.032143473625183105,1.2294319868087769,0.0018451482756063342,0.016877301037311554,-0.001730009913444519,-0.501085638999939,0.02749774605035782,-0.005966278724372387,-0.0049441163428127766,-0.08855222165584564,0.006622320972383022,0.04412432014942169,-0.027266837656497955,-1.1349124908447266,0.02287965640425682],"xaxis":"x","y":[-0.0020563285797834396,-0.0005101955030113459,0.0004685768508352339,0.00012511832755990326,-0.0006028721109032631,-0.00024295502225868404,-0.002318894723430276,-0.002758359769359231,0.0005645868368446827,0.000969740329310298,-0.0002504501899238676,4.740082658827305e-06,-0.0010071131400763988,0.0003947088844142854,-0.0015487205237150192,0.0014034901978448033,-0.0012652688892558217,-0.0011358250631019473,-0.0028159404173493385,-0.0029645359609276056,0.0029190238565206528,0.0025743518490344286,0.00036237656604498625,0.0017548884497955441,0.0005569332861341536,-0.00112663593608886,-0.0017354178708046675,-0.0014514722861349583,-0.00028740704874508083,0.0017210595542564988,0.00266590085811913,0.0031146793626248837,0.0005667305667884648,-0.0036664949730038643,-0.0018847067840397358,7.027178071439266e-06,-0.000726439815480262,0.00011369686399120837,0.001430142787285149,0.000749052269384265,0.0020184761378914118,0.0007436758605763316,-0.0004617759259417653,-0.003905785735696554,0.001140733016654849,-4.027335671707988e-05,-0.0013293256051838398,-0.0017636881675571203,-0.002828173339366913,0.0003364472358953208,-0.0014249038649722934,-0.0037773081567138433,0.0015998876187950373,0.00029889732832089067,-0.0008046309230849147,0.0020388164557516575,-0.0015593776479363441,-0.0006437147385440767,0.001116806990467012,-0.00035003889934159815,0.0011338151525706053,0.0011259106686338782,-0.0025163597892969847,-0.0014790240675210953,0.00038791983388364315,-6.407807813957334e-05,-0.0005096746608614922,-0.0008841876406222582,0.0006399309495463967,-0.001009696745313704,-0.0067590330727398396,0.0033667273819446564,-0.015147387981414795,-0.002135086804628372,0.002593189012259245,-0.00042674108408391476,-0.005559002980589867,0.0026659294962882996,0.006410874892026186,-0.0038270100485533476,-0.00038422830402851105,-0.0016430210089311004,-0.0013344308827072382,-9.184109512716532e-05,-9.488123760093004e-05,-0.005788922309875488,-0.0006383719155564904,0.134933739900589,-0.0017687628278508782,-0.018917974084615707,0.0038733629044145346,-0.002145076170563698,-0.010327237658202648,0.18325874209403992,-0.0007747883792035282,-0.0010452116839587688,-0.003833947703242302,-0.000804627372417599,-0.012673338875174522,0.008045812137424946,0.0036040153354406357,-0.009398169815540314,-0.08272106945514679,0.0035550352185964584,-0.018404126167297363,0.0017586719477549195,0.2896132469177246,0.022854045033454895,0.08595201373100281,-0.0006932567339390516,0.06816966831684113,0.01311141811311245,-0.021098004654049873,0.05112440511584282,1.384489893913269,0.04583733528852463,-0.038303446024656296,2.9854445457458496,0.001966139767318964,-0.008030213415622711,0.560872495174408,0.17083144187927246,-0.033618733286857605,0.05821547657251358,-0.0024530075024813414,0.0018771879840642214,0.2882729768753052,-1.898641586303711,-0.001528693363070488,-0.03513003885746002,0.48021769523620605,-0.0009116916917264462,0.0160758625715971,-0.03986112400889397,-0.3879111707210541,0.011123226955533028,-0.005477802362293005,-0.0025129495188593864,-0.08056114614009857,0.007518642581999302,0.043011054396629333,-0.0400824099779129,-0.9702335596084595,0.011862391605973244],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"Ablated"},"range":[-3,3]},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"Original"},"range":[-3,3]},"legend":{"tracegroupgap":0},"title":{"text":"Original vs Post-Ablation Direct Logit Attribution of Heads"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('aed2d9cd-0ab8-4309-b50e-32156504e9cc');
+var gd = document.getElementById('6e7a7d6f-0a22-4385-834f-25d48524e13a');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -2388,7 +2388,7 @@ <h3>Backup Name Mover Heads<a class="headerlink" href="#Backup-Name-Mover-Heads"
 </div>
 <p><strong>Exercise to the reader:</strong> Can you finish off this analysis? What’s going on here? Why are the backup name movers changing their behaviour? Why is one negative name mover becoming significantly less important?</p>
 <script type="application/vnd.jupyter.widget-state+json">
-{"state": {"f8f6869de10d4fbeb8b380d43cad31d4": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "4617f97d43ab4c59a4887b23130d1902": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "a05732c0ee01403fa202c360dc6bcc0b": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_f8f6869de10d4fbeb8b380d43cad31d4", "max": 548105171.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_4617f97d43ab4c59a4887b23130d1902", "tabbable": null, "tooltip": null, "value": 548105171.0}}, "17380503ad0e41f98634db778d6f7928": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "09c7749310034ebe804a345cddf59c63": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "5151e930cb894560a9f9046725028fde": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_17380503ad0e41f98634db778d6f7928", "placeholder": "\u200b", "style": "IPY_MODEL_09c7749310034ebe804a345cddf59c63", "tabbable": null, "tooltip": null, "value": "model.safetensors:\u2007100%"}}, "9751e1af80354cdd8adcd263569953b0": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "4c648589fd7b43ca80ac92f6ad748610": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "298f7209e9b24cb39e6d3b7c4079ff63": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_9751e1af80354cdd8adcd263569953b0", "placeholder": "\u200b", "style": "IPY_MODEL_4c648589fd7b43ca80ac92f6ad748610", "tabbable": null, "tooltip": null, "value": "\u2007548M/548M\u2007[00:03&lt;00:00,\u2007145MB/s]"}}, "96488d53528247589f8268eac0dd6b05": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "a42e2bbeb4644be780fd859fae156453": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_5151e930cb894560a9f9046725028fde", "IPY_MODEL_a05732c0ee01403fa202c360dc6bcc0b", "IPY_MODEL_298f7209e9b24cb39e6d3b7c4079ff63"], "layout": "IPY_MODEL_96488d53528247589f8268eac0dd6b05", "tabbable": null, "tooltip": null}}, "8476ed7c89264faa817152fd50d349a4": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "f31f8342fa4b494fa56fe0efd5549cdd": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "e2cc4fea81124e6b988f5ebdb35b756f": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_8476ed7c89264faa817152fd50d349a4", "max": 124.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_f31f8342fa4b494fa56fe0efd5549cdd", "tabbable": null, "tooltip": null, "value": 124.0}}, "235b0d59d9f64192ae6e01f1ab05fb5d": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "59dc2391b7964d3e9df04a6ba66f9ff1": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "737dbee311b44b2f813fbc7d1e1e643d": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_235b0d59d9f64192ae6e01f1ab05fb5d", "placeholder": "\u200b", "style": "IPY_MODEL_59dc2391b7964d3e9df04a6ba66f9ff1", "tabbable": null, "tooltip": null, "value": "generation_config.json:\u2007100%"}}, "ab8c1c1b39f948d29d4b0e0529342f27": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "c657cc795b0c46879505d7f6fa76f53e": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "36e2c3b1cbd74c4697352bef36d3cd2b": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_ab8c1c1b39f948d29d4b0e0529342f27", "placeholder": "\u200b", "style": "IPY_MODEL_c657cc795b0c46879505d7f6fa76f53e", "tabbable": null, "tooltip": null, "value": "\u2007124/124\u2007[00:00&lt;00:00,\u200721.8kB/s]"}}, "75ec75df34b544548416c31c31de1186": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "f98e0318bcde48c7bc9ff7fa4c098c64": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_737dbee311b44b2f813fbc7d1e1e643d", "IPY_MODEL_e2cc4fea81124e6b988f5ebdb35b756f", "IPY_MODEL_36e2c3b1cbd74c4697352bef36d3cd2b"], "layout": "IPY_MODEL_75ec75df34b544548416c31c31de1186", "tabbable": null, "tooltip": null}}, "74157bf082f24cc7b4a199d44fac71b7": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "094eda3d8c8e4ef4a71224ed2112888f": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "f45670d3a5504e4e8fb11ea1637ed193": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_74157bf082f24cc7b4a199d44fac71b7", "max": 26.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_094eda3d8c8e4ef4a71224ed2112888f", "tabbable": null, "tooltip": null, "value": 26.0}}, "327522f058604bc4a47401f696ced8de": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "8d1f546f867a4087a0467ce063107b44": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "68eaeabb3fba422a8d1bcd1cbb918576": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_327522f058604bc4a47401f696ced8de", "placeholder": "\u200b", "style": "IPY_MODEL_8d1f546f867a4087a0467ce063107b44", "tabbable": null, "tooltip": null, "value": "tokenizer_config.json:\u2007100%"}}, "da5b7f87aab84ec1a34aa52a7ad90661": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "d60f3ddba73a4d51b5f9a5458225d767": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "21341249f6ef4c12ac7114db65da28db": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_da5b7f87aab84ec1a34aa52a7ad90661", "placeholder": "\u200b", "style": "IPY_MODEL_d60f3ddba73a4d51b5f9a5458225d767", "tabbable": null, "tooltip": null, "value": "\u200726.0/26.0\u2007[00:00&lt;00:00,\u20074.56kB/s]"}}, "ef36e5539675442eba435bac9ff7cd7a": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "38bfcf873a8a4d7fbf3002998c2b9e9e": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_68eaeabb3fba422a8d1bcd1cbb918576", "IPY_MODEL_f45670d3a5504e4e8fb11ea1637ed193", "IPY_MODEL_21341249f6ef4c12ac7114db65da28db"], "layout": "IPY_MODEL_ef36e5539675442eba435bac9ff7cd7a", "tabbable": null, "tooltip": null}}, "2f286d154df947d38a5ae5ccea2b8fad": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "3de70caac20d4c1586ed10bac3217294": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "a59c6aa9b8ba42f5931d03545dd0b15a": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_2f286d154df947d38a5ae5ccea2b8fad", "max": 1042301.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_3de70caac20d4c1586ed10bac3217294", "tabbable": null, "tooltip": null, "value": 1042301.0}}, "972adad22a2847ea88250929516670dd": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "80570890418a4de6b5e46e26ad3eb27d": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "73d18ef08fc44b35878a0f0c98173961": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_972adad22a2847ea88250929516670dd", "placeholder": "\u200b", "style": "IPY_MODEL_80570890418a4de6b5e46e26ad3eb27d", "tabbable": null, "tooltip": null, "value": "vocab.json:\u2007100%"}}, "ad4c01375564485794658801d02bbb80": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "c57503aa752f4d889fc8fd1373ea4ad7": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "58e1228c5ada4acdb70c322dc097b209": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_ad4c01375564485794658801d02bbb80", "placeholder": "\u200b", "style": "IPY_MODEL_c57503aa752f4d889fc8fd1373ea4ad7", "tabbable": null, "tooltip": null, "value": "\u20071.04M/1.04M\u2007[00:00&lt;00:00,\u200712.2MB/s]"}}, "03f0891c87e343bcb0362f969ae5f46e": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "1184e705dd634ff29325f8222cc7406e": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_73d18ef08fc44b35878a0f0c98173961", "IPY_MODEL_a59c6aa9b8ba42f5931d03545dd0b15a", "IPY_MODEL_58e1228c5ada4acdb70c322dc097b209"], "layout": "IPY_MODEL_03f0891c87e343bcb0362f969ae5f46e", "tabbable": null, "tooltip": null}}, "abaf254a67b84797a0e9e6680b17bf84": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "f6fe61cf10ff4faca78694402dcc489a": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "a02b083035864edd890d8cc04587840d": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_abaf254a67b84797a0e9e6680b17bf84", "max": 456318.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_f6fe61cf10ff4faca78694402dcc489a", "tabbable": null, "tooltip": null, "value": 456318.0}}, "7a06d497954c45c2a940b32fc5efa389": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "ec4586211068491b9a0298a5323b0926": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "d4ce5dfcb1c64b039e68d2d08d7fa72c": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_7a06d497954c45c2a940b32fc5efa389", "placeholder": "\u200b", "style": "IPY_MODEL_ec4586211068491b9a0298a5323b0926", "tabbable": null, "tooltip": null, "value": "merges.txt:\u2007100%"}}, "20299bd8734d4ebe8b65a257c1124f15": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "49078358cf354e25b1d2c7f1b93c3d11": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "1fe6542d1e42421fab06b43a4911a479": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_20299bd8734d4ebe8b65a257c1124f15", "placeholder": "\u200b", "style": "IPY_MODEL_49078358cf354e25b1d2c7f1b93c3d11", "tabbable": null, "tooltip": null, "value": "\u2007456k/456k\u2007[00:00&lt;00:00,\u200752.5MB/s]"}}, "2e73aff0bf6244ad95040c4082ee0d74": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "b9a1fa123e9f4bd89dad50aac41fcff5": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_d4ce5dfcb1c64b039e68d2d08d7fa72c", "IPY_MODEL_a02b083035864edd890d8cc04587840d", "IPY_MODEL_1fe6542d1e42421fab06b43a4911a479"], "layout": "IPY_MODEL_2e73aff0bf6244ad95040c4082ee0d74", "tabbable": null, "tooltip": null}}, "6086e366713042c398600c11cb8615f2": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "adb3dd50e4e0478ebe832cbf0eda4c5d": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "782b31eb7e404113aadff4df80d6d284": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_6086e366713042c398600c11cb8615f2", "max": 1355256.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_adb3dd50e4e0478ebe832cbf0eda4c5d", "tabbable": null, "tooltip": null, "value": 1355256.0}}, "e9cb28639f7f44bfb5f15db88b8b9f32": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "73ac0538215847c6922f9a713c16f202": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "83098f01fbf74522b5034d9a0458a055": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_e9cb28639f7f44bfb5f15db88b8b9f32", "placeholder": "\u200b", "style": "IPY_MODEL_73ac0538215847c6922f9a713c16f202", "tabbable": null, "tooltip": null, "value": "tokenizer.json:\u2007100%"}}, "51a3da9a97644ba585cbdd682808c550": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "cb669f7d109c4532aabbce076c46d5c0": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "75f59c523d3d4af7bf07923f5c5b8a8d": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_51a3da9a97644ba585cbdd682808c550", "placeholder": "\u200b", "style": "IPY_MODEL_cb669f7d109c4532aabbce076c46d5c0", "tabbable": null, "tooltip": null, "value": "\u20071.36M/1.36M\u2007[00:00&lt;00:00,\u20077.22MB/s]"}}, "71a8d19ff6ba415fb452440886c5359f": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "b35dabf61035460aa1dd0200cf89fa93": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_83098f01fbf74522b5034d9a0458a055", "IPY_MODEL_782b31eb7e404113aadff4df80d6d284", "IPY_MODEL_75f59c523d3d4af7bf07923f5c5b8a8d"], "layout": "IPY_MODEL_71a8d19ff6ba415fb452440886c5359f", "tabbable": null, "tooltip": null}}}, "version_major": 2, "version_minor": 0}
+{"state": {"f36154b87b0644debdd37df866ca67a1": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "93b110135ece4b46a8b80e31468b2ab2": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "707dfb3454114fc7bac50f699e0fa74b": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_f36154b87b0644debdd37df866ca67a1", "max": 548105171.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_93b110135ece4b46a8b80e31468b2ab2", "tabbable": null, "tooltip": null, "value": 548105171.0}}, "db5d49ce14e24361b1be2eacc9479044": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "b5841c8cab9c4e97ba56ada551fdb2e0": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "812086c57ba84bab86d979d1da3bc3a6": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_db5d49ce14e24361b1be2eacc9479044", "placeholder": "\u200b", "style": "IPY_MODEL_b5841c8cab9c4e97ba56ada551fdb2e0", "tabbable": null, "tooltip": null, "value": "model.safetensors:\u2007100%"}}, "839d3a22e0b546d7bc98f515c83ca90c": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "f0f9e7c2bd7141449c36e529084c0b2b": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "e683fa7576544b28af3a08e8c34fdb6d": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_839d3a22e0b546d7bc98f515c83ca90c", "placeholder": "\u200b", "style": "IPY_MODEL_f0f9e7c2bd7141449c36e529084c0b2b", "tabbable": null, "tooltip": null, "value": "\u2007548M/548M\u2007[00:01&lt;00:00,\u2007287MB/s]"}}, "290303f0253140c1bd518e6e95f77b8d": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "515362cd489440da86ee9da4b18234d4": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_812086c57ba84bab86d979d1da3bc3a6", "IPY_MODEL_707dfb3454114fc7bac50f699e0fa74b", "IPY_MODEL_e683fa7576544b28af3a08e8c34fdb6d"], "layout": "IPY_MODEL_290303f0253140c1bd518e6e95f77b8d", "tabbable": null, "tooltip": null}}, "6b9a16110f364896abca2e2377be0a47": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "b3a416c3d9154ff7a1b2fb5ba1d1840c": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "58baa33b82404a59b690f3ad5d4a5bb1": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_6b9a16110f364896abca2e2377be0a47", "max": 124.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_b3a416c3d9154ff7a1b2fb5ba1d1840c", "tabbable": null, "tooltip": null, "value": 124.0}}, "e4aa1ab25fbc49fa96419a7f093c3e4e": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "0fbe54bdec604d568897b4cd3c043dad": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "993ed5ffb8e443a8837d959250db0fab": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_e4aa1ab25fbc49fa96419a7f093c3e4e", "placeholder": "\u200b", "style": "IPY_MODEL_0fbe54bdec604d568897b4cd3c043dad", "tabbable": null, "tooltip": null, "value": "generation_config.json:\u2007100%"}}, "f608c20d677340f2b49f090bb72868c0": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "340cbdd283674a878226f845ef97346b": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "6f5c1cf521994d24aa0ff115da16c22c": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_f608c20d677340f2b49f090bb72868c0", "placeholder": "\u200b", "style": "IPY_MODEL_340cbdd283674a878226f845ef97346b", "tabbable": null, "tooltip": null, "value": "\u2007124/124\u2007[00:00&lt;00:00,\u200721.4kB/s]"}}, "8fe8e25fa5794873b0b0b35a4745824d": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "6325ae084ee34b078afc0e06f01c6afa": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_993ed5ffb8e443a8837d959250db0fab", "IPY_MODEL_58baa33b82404a59b690f3ad5d4a5bb1", "IPY_MODEL_6f5c1cf521994d24aa0ff115da16c22c"], "layout": "IPY_MODEL_8fe8e25fa5794873b0b0b35a4745824d", "tabbable": null, "tooltip": null}}, "3105e32d09b24745b580cffafaa57174": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "7d80a0f33c4e48f0b0cef5bb8882aada": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "83ff07e9d88341fba479467e7cadc460": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_3105e32d09b24745b580cffafaa57174", "max": 26.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_7d80a0f33c4e48f0b0cef5bb8882aada", "tabbable": null, "tooltip": null, "value": 26.0}}, "5b43729786974ebf96e850b126e3b2dd": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "cce8300a7d9247859a7461581998e4ce": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "1d5596e7c1d84795868a2d32b89619ad": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_5b43729786974ebf96e850b126e3b2dd", "placeholder": "\u200b", "style": "IPY_MODEL_cce8300a7d9247859a7461581998e4ce", "tabbable": null, "tooltip": null, "value": "tokenizer_config.json:\u2007100%"}}, "a96b44caa83f45df904765efdc61793b": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "44a054ad7e3c4c59b2eeafe6008fa3dc": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "f5765110b6584f4bb805f71677f241b7": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_a96b44caa83f45df904765efdc61793b", "placeholder": "\u200b", "style": "IPY_MODEL_44a054ad7e3c4c59b2eeafe6008fa3dc", "tabbable": null, "tooltip": null, "value": "\u200726.0/26.0\u2007[00:00&lt;00:00,\u20074.39kB/s]"}}, "3901289b4d4a4337bb3bf0a7dcd8d5a4": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "12c7e81fccbe41dda672b6b6722e8f98": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_1d5596e7c1d84795868a2d32b89619ad", "IPY_MODEL_83ff07e9d88341fba479467e7cadc460", "IPY_MODEL_f5765110b6584f4bb805f71677f241b7"], "layout": "IPY_MODEL_3901289b4d4a4337bb3bf0a7dcd8d5a4", "tabbable": null, "tooltip": null}}, "48d38e46755540598972af4f8ffc5b83": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "0a4c123ed29c47bc87190f48ff5f90dd": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "aee08d1d53394415a2f7eb28925c5752": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_48d38e46755540598972af4f8ffc5b83", "max": 1042301.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_0a4c123ed29c47bc87190f48ff5f90dd", "tabbable": null, "tooltip": null, "value": 1042301.0}}, "80f8a6a0792d4a97ba6f083ae28b3218": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "8cbee9c01a5844769bc6ecac96eae6e9": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "921a2e40a82344bf99fbb499f83d755b": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_80f8a6a0792d4a97ba6f083ae28b3218", "placeholder": "\u200b", "style": "IPY_MODEL_8cbee9c01a5844769bc6ecac96eae6e9", "tabbable": null, "tooltip": null, "value": "vocab.json:\u2007100%"}}, "45eb78ab9b9c4813af6d66ce8726b8f4": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "f9f8d47155be4081b7f9ed66c6350cd6": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "de96e35e533a420c960013bfb1ead429": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_45eb78ab9b9c4813af6d66ce8726b8f4", "placeholder": "\u200b", "style": "IPY_MODEL_f9f8d47155be4081b7f9ed66c6350cd6", "tabbable": null, "tooltip": null, "value": "\u20071.04M/1.04M\u2007[00:00&lt;00:00,\u200736.1MB/s]"}}, "20480a53bb4e403fa078d8bf2fb4f0d1": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "f4d8cb4603f64164bba7cbecd423516d": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_921a2e40a82344bf99fbb499f83d755b", "IPY_MODEL_aee08d1d53394415a2f7eb28925c5752", "IPY_MODEL_de96e35e533a420c960013bfb1ead429"], "layout": "IPY_MODEL_20480a53bb4e403fa078d8bf2fb4f0d1", "tabbable": null, "tooltip": null}}, "eebc36e492664b00b22a70b251d2e44d": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "0251b82b322a4c019a7b949f50604da6": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "f5c4159733c54649893abf1ab34201c1": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_eebc36e492664b00b22a70b251d2e44d", "max": 456318.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_0251b82b322a4c019a7b949f50604da6", "tabbable": null, "tooltip": null, "value": 456318.0}}, "9872396553d24753a6a9f1b37df96713": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "c0896f84589a42d383d5b69e71979064": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "be2d8de653174dc184d20219e9716cba": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_9872396553d24753a6a9f1b37df96713", "placeholder": "\u200b", "style": "IPY_MODEL_c0896f84589a42d383d5b69e71979064", "tabbable": null, "tooltip": null, "value": "merges.txt:\u2007100%"}}, "2216c5f5b6444ff38fff530386df077f": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "5aa49a8508f84468b37311f1e3a7fa85": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "db72cb5fa9844c7b9087ff8d0e6e934c": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_2216c5f5b6444ff38fff530386df077f", "placeholder": "\u200b", "style": "IPY_MODEL_5aa49a8508f84468b37311f1e3a7fa85", "tabbable": null, "tooltip": null, "value": "\u2007456k/456k\u2007[00:00&lt;00:00,\u200765.2MB/s]"}}, "2fd6e310c91a4be4af4e44f2ac0d65e4": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "27abca058a2d442c9fa1488c14f5fe1c": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_be2d8de653174dc184d20219e9716cba", "IPY_MODEL_f5c4159733c54649893abf1ab34201c1", "IPY_MODEL_db72cb5fa9844c7b9087ff8d0e6e934c"], "layout": "IPY_MODEL_2fd6e310c91a4be4af4e44f2ac0d65e4", "tabbable": null, "tooltip": null}}, "386bba55285147a4aedf17cb5a7e0215": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "30f82becd85f4ab6b8816b9312d1cdd5": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "d0a71171b9454878a990f89c1e164277": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_386bba55285147a4aedf17cb5a7e0215", "max": 1355256.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_30f82becd85f4ab6b8816b9312d1cdd5", "tabbable": null, "tooltip": null, "value": 1355256.0}}, "30b072ac28a94fa0a11b9803daddea18": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "a49e011ea5374814b360bde1cf84c71a": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "894f3b3042554c25aa4ceacd228af5c2": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_30b072ac28a94fa0a11b9803daddea18", "placeholder": "\u200b", "style": "IPY_MODEL_a49e011ea5374814b360bde1cf84c71a", "tabbable": null, "tooltip": null, "value": "tokenizer.json:\u2007100%"}}, "46051a9cf3c747f8a5ec0234ff42bf09": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "9780b9d58ebe4211a7a618aa44328af9": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "ef62f1509335401ea8b186ea892dcf60": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_46051a9cf3c747f8a5ec0234ff42bf09", "placeholder": "\u200b", "style": "IPY_MODEL_9780b9d58ebe4211a7a618aa44328af9", "tabbable": null, "tooltip": null, "value": "\u20071.36M/1.36M\u2007[00:00&lt;00:00,\u200732.5MB/s]"}}, "ffbeb3144f08405e8289aaed02f79c8a": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "f10863d1bbd14b91819db0b9074aa85a": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_894f3b3042554c25aa4ceacd228af5c2", "IPY_MODEL_d0a71171b9454878a990f89c1e164277", "IPY_MODEL_ef62f1509335401ea8b186ea892dcf60"], "layout": "IPY_MODEL_ffbeb3144f08405e8289aaed02f79c8a", "tabbable": null, "tooltip": null}}}, "version_major": 2, "version_minor": 0}
 </script></section>
 </section>
 </section>
diff --git a/generated/demos/Exploratory_Analysis_Demo.ipynb b/generated/demos/Exploratory_Analysis_Demo.ipynb
index 2949525bd..1943c305b 100644
--- a/generated/demos/Exploratory_Analysis_Demo.ipynb
+++ b/generated/demos/Exploratory_Analysis_Demo.ipynb
@@ -65,10 +65,10 @@
    "execution_count": 1,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:37:06.258245Z",
-     "iopub.status.busy": "2024-09-05T17:37:06.257825Z",
-     "iopub.status.idle": "2024-09-05T17:37:06.268299Z",
-     "shell.execute_reply": "2024-09-05T17:37:06.267851Z"
+     "iopub.execute_input": "2024-09-10T17:10:52.122808Z",
+     "iopub.status.busy": "2024-09-10T17:10:52.122638Z",
+     "iopub.status.idle": "2024-09-10T17:10:52.132262Z",
+     "shell.execute_reply": "2024-09-10T17:10:52.131699Z"
     }
    },
    "outputs": [],
@@ -110,10 +110,10 @@
    "execution_count": 2,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:37:06.270516Z",
-     "iopub.status.busy": "2024-09-05T17:37:06.270156Z",
-     "iopub.status.idle": "2024-09-05T17:37:09.631700Z",
-     "shell.execute_reply": "2024-09-05T17:37:09.631121Z"
+     "iopub.execute_input": "2024-09-10T17:10:52.134520Z",
+     "iopub.status.busy": "2024-09-10T17:10:52.134180Z",
+     "iopub.status.idle": "2024-09-10T17:10:55.411845Z",
+     "shell.execute_reply": "2024-09-10T17:10:55.411264Z"
     }
    },
    "outputs": [],
@@ -154,10 +154,10 @@
    "execution_count": 3,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:37:09.634661Z",
-     "iopub.status.busy": "2024-09-05T17:37:09.634137Z",
-     "iopub.status.idle": "2024-09-05T17:37:09.637722Z",
-     "shell.execute_reply": "2024-09-05T17:37:09.637100Z"
+     "iopub.execute_input": "2024-09-10T17:10:55.414417Z",
+     "iopub.status.busy": "2024-09-10T17:10:55.414133Z",
+     "iopub.status.idle": "2024-09-10T17:10:55.417200Z",
+     "shell.execute_reply": "2024-09-10T17:10:55.416684Z"
     }
    },
    "outputs": [
@@ -193,10 +193,10 @@
    "execution_count": 4,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:37:09.672026Z",
-     "iopub.status.busy": "2024-09-05T17:37:09.671459Z",
-     "iopub.status.idle": "2024-09-05T17:37:09.676318Z",
-     "shell.execute_reply": "2024-09-05T17:37:09.675715Z"
+     "iopub.execute_input": "2024-09-10T17:10:55.450845Z",
+     "iopub.status.busy": "2024-09-10T17:10:55.450452Z",
+     "iopub.status.idle": "2024-09-10T17:10:55.454956Z",
+     "shell.execute_reply": "2024-09-10T17:10:55.454381Z"
     }
    },
    "outputs": [],
@@ -281,17 +281,17 @@
    "execution_count": 5,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:37:09.678608Z",
-     "iopub.status.busy": "2024-09-05T17:37:09.678190Z",
-     "iopub.status.idle": "2024-09-05T17:37:17.797314Z",
-     "shell.execute_reply": "2024-09-05T17:37:17.796772Z"
+     "iopub.execute_input": "2024-09-10T17:10:55.457088Z",
+     "iopub.status.busy": "2024-09-10T17:10:55.456913Z",
+     "iopub.status.idle": "2024-09-10T17:10:59.991168Z",
+     "shell.execute_reply": "2024-09-10T17:10:59.990533Z"
     }
    },
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "a42e2bbeb4644be780fd859fae156453",
+       "model_id": "515362cd489440da86ee9da4b18234d4",
        "version_major": 2,
        "version_minor": 0
       },
@@ -305,7 +305,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "f98e0318bcde48c7bc9ff7fa4c098c64",
+       "model_id": "6325ae084ee34b078afc0e06f01c6afa",
        "version_major": 2,
        "version_minor": 0
       },
@@ -319,7 +319,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "38bfcf873a8a4d7fbf3002998c2b9e9e",
+       "model_id": "12c7e81fccbe41dda672b6b6722e8f98",
        "version_major": 2,
        "version_minor": 0
       },
@@ -333,7 +333,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "1184e705dd634ff29325f8222cc7406e",
+       "model_id": "f4d8cb4603f64164bba7cbecd423516d",
        "version_major": 2,
        "version_minor": 0
       },
@@ -347,7 +347,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "b9a1fa123e9f4bd89dad50aac41fcff5",
+       "model_id": "27abca058a2d442c9fa1488c14f5fe1c",
        "version_major": 2,
        "version_minor": 0
       },
@@ -361,7 +361,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "b35dabf61035460aa1dd0200cf89fa93",
+       "model_id": "f10863d1bbd14b91819db0b9074aa85a",
        "version_major": 2,
        "version_minor": 0
       },
@@ -413,10 +413,10 @@
    "execution_count": 6,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:37:17.799537Z",
-     "iopub.status.busy": "2024-09-05T17:37:17.799342Z",
-     "iopub.status.idle": "2024-09-05T17:37:18.164532Z",
-     "shell.execute_reply": "2024-09-05T17:37:18.163911Z"
+     "iopub.execute_input": "2024-09-10T17:10:59.993773Z",
+     "iopub.status.busy": "2024-09-10T17:10:59.993381Z",
+     "iopub.status.idle": "2024-09-10T17:11:00.363935Z",
+     "shell.execute_reply": "2024-09-10T17:11:00.363304Z"
     }
    },
    "outputs": [
@@ -501,10 +501,10 @@
    "execution_count": 7,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:37:18.167228Z",
-     "iopub.status.busy": "2024-09-05T17:37:18.166763Z",
-     "iopub.status.idle": "2024-09-05T17:37:18.174464Z",
-     "shell.execute_reply": "2024-09-05T17:37:18.173964Z"
+     "iopub.execute_input": "2024-09-10T17:11:00.366063Z",
+     "iopub.status.busy": "2024-09-10T17:11:00.365878Z",
+     "iopub.status.idle": "2024-09-10T17:11:00.373006Z",
+     "shell.execute_reply": "2024-09-10T17:11:00.372556Z"
     }
    },
    "outputs": [
@@ -569,10 +569,10 @@
    "execution_count": 8,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:37:18.176612Z",
-     "iopub.status.busy": "2024-09-05T17:37:18.176266Z",
-     "iopub.status.idle": "2024-09-05T17:37:18.182608Z",
-     "shell.execute_reply": "2024-09-05T17:37:18.182113Z"
+     "iopub.execute_input": "2024-09-10T17:11:00.374930Z",
+     "iopub.status.busy": "2024-09-10T17:11:00.374753Z",
+     "iopub.status.idle": "2024-09-10T17:11:00.381018Z",
+     "shell.execute_reply": "2024-09-10T17:11:00.380444Z"
     }
    },
    "outputs": [
@@ -618,10 +618,10 @@
    "execution_count": 9,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:37:18.184756Z",
-     "iopub.status.busy": "2024-09-05T17:37:18.184422Z",
-     "iopub.status.idle": "2024-09-05T17:37:18.470008Z",
-     "shell.execute_reply": "2024-09-05T17:37:18.469354Z"
+     "iopub.execute_input": "2024-09-10T17:11:00.382932Z",
+     "iopub.status.busy": "2024-09-10T17:11:00.382763Z",
+     "iopub.status.idle": "2024-09-10T17:11:00.676617Z",
+     "shell.execute_reply": "2024-09-10T17:11:00.675965Z"
     }
    },
    "outputs": [],
@@ -644,10 +644,10 @@
    "execution_count": 10,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:37:18.472827Z",
-     "iopub.status.busy": "2024-09-05T17:37:18.472476Z",
-     "iopub.status.idle": "2024-09-05T17:37:18.478239Z",
-     "shell.execute_reply": "2024-09-05T17:37:18.477746Z"
+     "iopub.execute_input": "2024-09-10T17:11:00.679455Z",
+     "iopub.status.busy": "2024-09-10T17:11:00.679233Z",
+     "iopub.status.idle": "2024-09-10T17:11:00.685095Z",
+     "shell.execute_reply": "2024-09-10T17:11:00.684624Z"
     }
    },
    "outputs": [
@@ -788,10 +788,10 @@
    "execution_count": 11,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:37:18.480500Z",
-     "iopub.status.busy": "2024-09-05T17:37:18.480175Z",
-     "iopub.status.idle": "2024-09-05T17:37:18.484095Z",
-     "shell.execute_reply": "2024-09-05T17:37:18.483517Z"
+     "iopub.execute_input": "2024-09-10T17:11:00.687271Z",
+     "iopub.status.busy": "2024-09-10T17:11:00.686963Z",
+     "iopub.status.idle": "2024-09-10T17:11:00.690943Z",
+     "shell.execute_reply": "2024-09-10T17:11:00.690442Z"
     }
    },
    "outputs": [
@@ -839,10 +839,10 @@
    "execution_count": 12,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:37:18.486349Z",
-     "iopub.status.busy": "2024-09-05T17:37:18.486008Z",
-     "iopub.status.idle": "2024-09-05T17:37:18.490882Z",
-     "shell.execute_reply": "2024-09-05T17:37:18.490272Z"
+     "iopub.execute_input": "2024-09-10T17:11:00.693096Z",
+     "iopub.status.busy": "2024-09-10T17:11:00.692733Z",
+     "iopub.status.idle": "2024-09-10T17:11:00.698594Z",
+     "shell.execute_reply": "2024-09-10T17:11:00.698155Z"
     }
    },
    "outputs": [
@@ -895,10 +895,10 @@
    "execution_count": 13,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:37:18.492963Z",
-     "iopub.status.busy": "2024-09-05T17:37:18.492652Z",
-     "iopub.status.idle": "2024-09-05T17:37:18.495949Z",
-     "shell.execute_reply": "2024-09-05T17:37:18.495478Z"
+     "iopub.execute_input": "2024-09-10T17:11:00.700697Z",
+     "iopub.status.busy": "2024-09-10T17:11:00.700363Z",
+     "iopub.status.idle": "2024-09-10T17:11:00.703450Z",
+     "shell.execute_reply": "2024-09-10T17:11:00.702982Z"
     }
    },
    "outputs": [],
@@ -940,10 +940,10 @@
    "execution_count": 14,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:37:18.497800Z",
-     "iopub.status.busy": "2024-09-05T17:37:18.497631Z",
-     "iopub.status.idle": "2024-09-05T17:37:19.293415Z",
-     "shell.execute_reply": "2024-09-05T17:37:19.292749Z"
+     "iopub.execute_input": "2024-09-10T17:11:00.705231Z",
+     "iopub.status.busy": "2024-09-10T17:11:00.705075Z",
+     "iopub.status.idle": "2024-09-10T17:11:01.460245Z",
+     "shell.execute_reply": "2024-09-10T17:11:01.459557Z"
     }
    },
    "outputs": [
@@ -1929,9 +1929,9 @@
        }
       },
       "text/html": [
-       "<div>                            <div id=\"54c9da27-b5e5-42aa-baed-1fe18fb97e41\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"54c9da27-b5e5-42aa-baed-1fe18fb97e41\")) {                    Plotly.newPlot(                        \"54c9da27-b5e5-42aa-baed-1fe18fb97e41\",                        [{\"hovertemplate\":\"\\u003cb\\u003e%{hovertext}\\u003c\\u002fb\\u003e\\u003cbr\\u003e\\u003cbr\\u003ex=%{x}\\u003cbr\\u003ey=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"hovertext\":[\"0_pre\",\"0_mid\",\"1_pre\",\"1_mid\",\"2_pre\",\"2_mid\",\"3_pre\",\"3_mid\",\"4_pre\",\"4_mid\",\"5_pre\",\"5_mid\",\"6_pre\",\"6_mid\",\"7_pre\",\"7_mid\",\"8_pre\",\"8_mid\",\"9_pre\",\"9_mid\",\"10_pre\",\"10_mid\",\"11_pre\",\"11_mid\",\"final_post\"],\"legendgroup\":\"\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"\",\"orientation\":\"v\",\"showlegend\":false,\"x\":[0.0,0.5,1.0,1.5,2.0,2.5,3.0,3.5,4.0,4.5,5.0,5.5,6.0,6.5,7.0,7.5,8.0,8.5,9.0,9.5,10.0,10.5,11.0,11.5,12.0],\"xaxis\":\"x\",\"y\":[1.2936594430357218e-05,-0.0066434419713914394,-0.007525080814957619,-0.009075661189854145,-0.008736724965274334,-0.008685516193509102,-0.006480449344962835,-0.007939981296658516,-0.009662019088864326,-0.01509616058319807,-0.014190619811415672,-0.019929546862840652,-0.009124485775828362,-0.02729814499616623,-0.02985497936606407,0.24972598254680634,0.25055843591690063,0.4500514566898346,0.4599689245223999,5.025448322296143,5.142899036407471,4.73056697845459,4.887067794799805,3.4453935623168945,3.5518839359283447],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"x\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"y\"}},\"legend\":{\"tracegroupgap\":0},\"title\":{\"text\":\"Logit Difference From Accumulate Residual Stream\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "<div>                            <div id=\"3d63aabd-d814-4fc0-b7e4-841c761b56cd\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"3d63aabd-d814-4fc0-b7e4-841c761b56cd\")) {                    Plotly.newPlot(                        \"3d63aabd-d814-4fc0-b7e4-841c761b56cd\",                        [{\"hovertemplate\":\"\\u003cb\\u003e%{hovertext}\\u003c\\u002fb\\u003e\\u003cbr\\u003e\\u003cbr\\u003ex=%{x}\\u003cbr\\u003ey=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"hovertext\":[\"0_pre\",\"0_mid\",\"1_pre\",\"1_mid\",\"2_pre\",\"2_mid\",\"3_pre\",\"3_mid\",\"4_pre\",\"4_mid\",\"5_pre\",\"5_mid\",\"6_pre\",\"6_mid\",\"7_pre\",\"7_mid\",\"8_pre\",\"8_mid\",\"9_pre\",\"9_mid\",\"10_pre\",\"10_mid\",\"11_pre\",\"11_mid\",\"final_post\"],\"legendgroup\":\"\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"\",\"orientation\":\"v\",\"showlegend\":false,\"x\":[0.0,0.5,1.0,1.5,2.0,2.5,3.0,3.5,4.0,4.5,5.0,5.5,6.0,6.5,7.0,7.5,8.0,8.5,9.0,9.5,10.0,10.5,11.0,11.5,12.0],\"xaxis\":\"x\",\"y\":[1.2936594430357218e-05,-0.0066434419713914394,-0.007525080814957619,-0.009075661189854145,-0.008736724965274334,-0.008685516193509102,-0.006480449344962835,-0.007939981296658516,-0.009662019088864326,-0.01509616058319807,-0.014190619811415672,-0.019929546862840652,-0.009124485775828362,-0.02729814499616623,-0.02985497936606407,0.24972598254680634,0.25055843591690063,0.4500514566898346,0.4599689245223999,5.025448322296143,5.142899036407471,4.73056697845459,4.887067794799805,3.4453935623168945,3.5518839359283447],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"x\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"y\"}},\"legend\":{\"tracegroupgap\":0},\"title\":{\"text\":\"Logit Difference From Accumulate Residual Stream\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('54c9da27-b5e5-42aa-baed-1fe18fb97e41');\n",
+       "var gd = document.getElementById('3d63aabd-d814-4fc0-b7e4-841c761b56cd');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -1996,10 +1996,10 @@
    "execution_count": 15,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:37:19.331984Z",
-     "iopub.status.busy": "2024-09-05T17:37:19.331565Z",
-     "iopub.status.idle": "2024-09-05T17:37:19.366692Z",
-     "shell.execute_reply": "2024-09-05T17:37:19.366080Z"
+     "iopub.execute_input": "2024-09-10T17:11:01.499757Z",
+     "iopub.status.busy": "2024-09-10T17:11:01.499262Z",
+     "iopub.status.idle": "2024-09-10T17:11:01.533757Z",
+     "shell.execute_reply": "2024-09-10T17:11:01.533192Z"
     }
    },
    "outputs": [
@@ -2959,9 +2959,9 @@
        }
       },
       "text/html": [
-       "<div>                            <div id=\"13c88f1d-dc19-46a7-ac53-d34066484b06\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"13c88f1d-dc19-46a7-ac53-d34066484b06\")) {                    Plotly.newPlot(                        \"13c88f1d-dc19-46a7-ac53-d34066484b06\",                        [{\"hovertemplate\":\"\\u003cb\\u003e%{hovertext}\\u003c\\u002fb\\u003e\\u003cbr\\u003e\\u003cbr\\u003ex=%{x}\\u003cbr\\u003ey=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"hovertext\":[\"embed\",\"pos_embed\",\"0_attn_out\",\"0_mlp_out\",\"1_attn_out\",\"1_mlp_out\",\"2_attn_out\",\"2_mlp_out\",\"3_attn_out\",\"3_mlp_out\",\"4_attn_out\",\"4_mlp_out\",\"5_attn_out\",\"5_mlp_out\",\"6_attn_out\",\"6_mlp_out\",\"7_attn_out\",\"7_mlp_out\",\"8_attn_out\",\"8_mlp_out\",\"9_attn_out\",\"9_mlp_out\",\"10_attn_out\",\"10_mlp_out\",\"11_attn_out\",\"11_mlp_out\"],\"legendgroup\":\"\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"\",\"orientation\":\"v\",\"showlegend\":false,\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25],\"xaxis\":\"x\",\"y\":[-0.00028366505284793675,0.00029660051222890615,-0.006656361743807793,-0.0008817464113235474,-0.0015505198389291763,0.0003389097983017564,5.121063441038132e-05,0.002205097349360585,-0.001459577470086515,-0.0017220661975443363,-0.0054340846836566925,0.0009054935071617365,-0.005738964769989252,0.010805057361721992,-0.018173594027757645,-0.002556750550866127,0.27958089113235474,0.0008324328809976578,0.19949296116828918,0.009917395189404488,4.565478801727295,0.11744903028011322,-0.41232579946517944,0.1564953327178955,-1.4416704177856445,0.10648898035287857],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"x\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"y\"}},\"legend\":{\"tracegroupgap\":0},\"title\":{\"text\":\"Logit Difference From Each Layer\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "<div>                            <div id=\"8309d7f8-599e-48e5-859f-a6ec1ae1a080\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"8309d7f8-599e-48e5-859f-a6ec1ae1a080\")) {                    Plotly.newPlot(                        \"8309d7f8-599e-48e5-859f-a6ec1ae1a080\",                        [{\"hovertemplate\":\"\\u003cb\\u003e%{hovertext}\\u003c\\u002fb\\u003e\\u003cbr\\u003e\\u003cbr\\u003ex=%{x}\\u003cbr\\u003ey=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"hovertext\":[\"embed\",\"pos_embed\",\"0_attn_out\",\"0_mlp_out\",\"1_attn_out\",\"1_mlp_out\",\"2_attn_out\",\"2_mlp_out\",\"3_attn_out\",\"3_mlp_out\",\"4_attn_out\",\"4_mlp_out\",\"5_attn_out\",\"5_mlp_out\",\"6_attn_out\",\"6_mlp_out\",\"7_attn_out\",\"7_mlp_out\",\"8_attn_out\",\"8_mlp_out\",\"9_attn_out\",\"9_mlp_out\",\"10_attn_out\",\"10_mlp_out\",\"11_attn_out\",\"11_mlp_out\"],\"legendgroup\":\"\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"\",\"orientation\":\"v\",\"showlegend\":false,\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25],\"xaxis\":\"x\",\"y\":[-0.00028366505284793675,0.00029660051222890615,-0.006656361743807793,-0.0008817464113235474,-0.0015505198389291763,0.0003389097983017564,5.121063441038132e-05,0.002205097349360585,-0.001459577470086515,-0.0017220661975443363,-0.0054340846836566925,0.0009054935071617365,-0.005738964769989252,0.010805057361721992,-0.018173594027757645,-0.002556750550866127,0.27958089113235474,0.0008324328809976578,0.19949296116828918,0.009917395189404488,4.565478801727295,0.11744903028011322,-0.41232579946517944,0.1564953327178955,-1.4416704177856445,0.10648898035287857],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"x\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"y\"}},\"legend\":{\"tracegroupgap\":0},\"title\":{\"text\":\"Logit Difference From Each Layer\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('13c88f1d-dc19-46a7-ac53-d34066484b06');\n",
+       "var gd = document.getElementById('8309d7f8-599e-48e5-859f-a6ec1ae1a080');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -3025,10 +3025,10 @@
    "execution_count": 16,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:37:19.369086Z",
-     "iopub.status.busy": "2024-09-05T17:37:19.368745Z",
-     "iopub.status.idle": "2024-09-05T17:37:19.435834Z",
-     "shell.execute_reply": "2024-09-05T17:37:19.435180Z"
+     "iopub.execute_input": "2024-09-10T17:11:01.535992Z",
+     "iopub.status.busy": "2024-09-10T17:11:01.535632Z",
+     "iopub.status.idle": "2024-09-10T17:11:01.597678Z",
+     "shell.execute_reply": "2024-09-10T17:11:01.597059Z"
     }
    },
    "outputs": [
@@ -4121,9 +4121,9 @@
        }
       },
       "text/html": [
-       "<div>                            <div id=\"c34feb68-85f6-4bc2-84d3-a7697a1138a3\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"c34feb68-85f6-4bc2-84d3-a7697a1138a3\")) {                    Plotly.newPlot(                        \"c34feb68-85f6-4bc2-84d3-a7697a1138a3\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[-0.0020563285797834396,-0.0005101955030113459,0.0004685768508352339,0.00012511832755990326,-0.0006028721109032631,-0.00024295502225868404,-0.002318894723430276,-0.002758359769359231,0.0005645868368446827,0.000969740329310298,-0.0002504501899238676,4.740082658827305e-06],[-0.0010071131400763988,0.0003947088844142854,-0.0015487205237150192,0.0014034901978448033,-0.0012652688892558217,-0.0011358250631019473,-0.0028159404173493385,-0.0029645359609276056,0.0029190238565206528,0.0025743518490344286,0.00036237656604498625,0.0017548884497955441],[0.0005569332861341536,-0.00112663593608886,-0.0017354178708046675,-0.0014514722861349583,-0.00028740704874508083,0.0017210595542564988,0.00266590085811913,0.0031146793626248837,0.0005667305667884648,-0.0036664949730038643,-0.0018847067840397358,7.027178071439266e-06],[-0.000726439815480262,0.00011369686399120837,0.001430142787285149,0.000749052269384265,0.0020184761378914118,0.0007436758605763316,-0.0004617759259417653,-0.003905785735696554,0.001140733016654849,-4.027335671707988e-05,-0.0013293256051838398,-0.0017636881675571203],[-0.002828173339366913,0.0003364472358953208,-0.0014249038649722934,-0.0037773081567138433,0.0015998876187950373,0.00029889732832089067,-0.0008046309230849147,0.0020388164557516575,-0.0015593776479363441,-0.0006437147385440767,0.001116806990467012,-0.00035003889934159815],[0.0011338151525706053,0.0011259106686338782,-0.0025163597892969847,-0.0014790240675210953,0.00038791983388364315,-6.407807813957334e-05,-0.0005096746608614922,-0.0008841876406222582,0.0006399309495463967,-0.001009696745313704,-0.0067590330727398396,0.0033667273819446564],[-0.015147387981414795,-0.002135086804628372,0.002593189012259245,-0.00042674108408391476,-0.005559002980589867,0.0026659294962882996,0.006410874892026186,-0.0038270100485533476,-0.00038422830402851105,-0.0016430210089311004,-0.0013344308827072382,-9.184109512716532e-05],[-9.488123760093004e-05,-0.005788922309875488,-0.0006383719155564904,0.134933739900589,-0.0017687628278508782,-0.018917974084615707,0.0038733629044145346,-0.002145076170563698,-0.010327237658202648,0.18325874209403992,-0.0007747883792035282,-0.0010452116839587688],[-0.003833947703242302,-0.000804627372417599,-0.012673338875174522,0.008045812137424946,0.0036040153354406357,-0.009398169815540314,-0.08272106945514679,0.0035550352185964584,-0.018404126167297363,0.0017586719477549195,0.2896132469177246,0.022854045033454895],[0.08595201373100281,-0.0006932567339390516,0.06816966831684113,0.01311141811311245,-0.021098004654049873,0.05112440511584282,1.384489893913269,0.04583733528852463,-0.038303446024656296,2.9854445457458496,0.001966139767318964,-0.008030213415622711],[0.560872495174408,0.17083144187927246,-0.033618733286857605,0.05821547657251358,-0.0024530075024813414,0.0018771879840642214,0.2882729768753052,-1.898641586303711,-0.001528693363070488,-0.03513003885746002,0.48021769523620605,-0.0009116916917264462],[0.0160758625715971,-0.03986112400889397,-0.3879111707210541,0.011123226955533028,-0.005477802362293005,-0.0025129495188593864,-0.08056114614009857,0.007518642581999302,0.043011054396629333,-0.0400824099779129,-0.9702335596084595,0.011862391605973244]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Logit Difference From Each Head\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "<div>                            <div id=\"3827203b-16c0-45c2-aab4-743cf2cb601f\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"3827203b-16c0-45c2-aab4-743cf2cb601f\")) {                    Plotly.newPlot(                        \"3827203b-16c0-45c2-aab4-743cf2cb601f\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[-0.0020563285797834396,-0.0005101955030113459,0.0004685768508352339,0.00012511832755990326,-0.0006028721109032631,-0.00024295502225868404,-0.002318894723430276,-0.002758359769359231,0.0005645868368446827,0.000969740329310298,-0.0002504501899238676,4.740082658827305e-06],[-0.0010071131400763988,0.0003947088844142854,-0.0015487205237150192,0.0014034901978448033,-0.0012652688892558217,-0.0011358250631019473,-0.0028159404173493385,-0.0029645359609276056,0.0029190238565206528,0.0025743518490344286,0.00036237656604498625,0.0017548884497955441],[0.0005569332861341536,-0.00112663593608886,-0.0017354178708046675,-0.0014514722861349583,-0.00028740704874508083,0.0017210595542564988,0.00266590085811913,0.0031146793626248837,0.0005667305667884648,-0.0036664949730038643,-0.0018847067840397358,7.027178071439266e-06],[-0.000726439815480262,0.00011369686399120837,0.001430142787285149,0.000749052269384265,0.0020184761378914118,0.0007436758605763316,-0.0004617759259417653,-0.003905785735696554,0.001140733016654849,-4.027335671707988e-05,-0.0013293256051838398,-0.0017636881675571203],[-0.002828173339366913,0.0003364472358953208,-0.0014249038649722934,-0.0037773081567138433,0.0015998876187950373,0.00029889732832089067,-0.0008046309230849147,0.0020388164557516575,-0.0015593776479363441,-0.0006437147385440767,0.001116806990467012,-0.00035003889934159815],[0.0011338151525706053,0.0011259106686338782,-0.0025163597892969847,-0.0014790240675210953,0.00038791983388364315,-6.407807813957334e-05,-0.0005096746608614922,-0.0008841876406222582,0.0006399309495463967,-0.001009696745313704,-0.0067590330727398396,0.0033667273819446564],[-0.015147387981414795,-0.002135086804628372,0.002593189012259245,-0.00042674108408391476,-0.005559002980589867,0.0026659294962882996,0.006410874892026186,-0.0038270100485533476,-0.00038422830402851105,-0.0016430210089311004,-0.0013344308827072382,-9.184109512716532e-05],[-9.488123760093004e-05,-0.005788922309875488,-0.0006383719155564904,0.134933739900589,-0.0017687628278508782,-0.018917974084615707,0.0038733629044145346,-0.002145076170563698,-0.010327237658202648,0.18325874209403992,-0.0007747883792035282,-0.0010452116839587688],[-0.003833947703242302,-0.000804627372417599,-0.012673338875174522,0.008045812137424946,0.0036040153354406357,-0.009398169815540314,-0.08272106945514679,0.0035550352185964584,-0.018404126167297363,0.0017586719477549195,0.2896132469177246,0.022854045033454895],[0.08595201373100281,-0.0006932567339390516,0.06816966831684113,0.01311141811311245,-0.021098004654049873,0.05112440511584282,1.384489893913269,0.04583733528852463,-0.038303446024656296,2.9854445457458496,0.001966139767318964,-0.008030213415622711],[0.560872495174408,0.17083144187927246,-0.033618733286857605,0.05821547657251358,-0.0024530075024813414,0.0018771879840642214,0.2882729768753052,-1.898641586303711,-0.001528693363070488,-0.03513003885746002,0.48021769523620605,-0.0009116916917264462],[0.0160758625715971,-0.03986112400889397,-0.3879111707210541,0.011123226955533028,-0.005477802362293005,-0.0025129495188593864,-0.08056114614009857,0.007518642581999302,0.043011054396629333,-0.0400824099779129,-0.9702335596084595,0.011862391605973244]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Logit Difference From Each Head\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('c34feb68-85f6-4bc2-84d3-a7697a1138a3');\n",
+       "var gd = document.getElementById('3827203b-16c0-45c2-aab4-743cf2cb601f');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -4195,10 +4195,10 @@
    "execution_count": 17,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:37:19.438077Z",
-     "iopub.status.busy": "2024-09-05T17:37:19.437893Z",
-     "iopub.status.idle": "2024-09-05T17:37:19.443279Z",
-     "shell.execute_reply": "2024-09-05T17:37:19.442821Z"
+     "iopub.execute_input": "2024-09-10T17:11:01.599789Z",
+     "iopub.status.busy": "2024-09-10T17:11:01.599579Z",
+     "iopub.status.idle": "2024-09-10T17:11:01.604880Z",
+     "shell.execute_reply": "2024-09-10T17:11:01.604431Z"
     }
    },
    "outputs": [],
@@ -4263,29 +4263,29 @@
    "execution_count": 18,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:37:19.445191Z",
-     "iopub.status.busy": "2024-09-05T17:37:19.445019Z",
-     "iopub.status.idle": "2024-09-05T17:37:19.455165Z",
-     "shell.execute_reply": "2024-09-05T17:37:19.454576Z"
+     "iopub.execute_input": "2024-09-10T17:11:01.606705Z",
+     "iopub.status.busy": "2024-09-10T17:11:01.606538Z",
+     "iopub.status.idle": "2024-09-10T17:11:01.616165Z",
+     "shell.execute_reply": "2024-09-10T17:11:01.615618Z"
     }
    },
    "outputs": [
     {
      "data": {
       "text/html": [
-       "<div style='max-width: 700px;'><h2>Top 3 Positive Logit Attribution Heads</h2><br/><div id=\"circuits-vis-6d53690c-9574\" style=\"margin: 15px 0;\"/>\n",
+       "<div style='max-width: 700px;'><h2>Top 3 Positive Logit Attribution Heads</h2><br/><div id=\"circuits-vis-640204d8-20cc\" style=\"margin: 15px 0;\"/>\n",
        "    <script crossorigin type=\"module\">\n",
        "    import { render, AttentionHeads } from \"https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js\";\n",
        "    render(\n",
-       "      \"circuits-vis-6d53690c-9574\",\n",
+       "      \"circuits-vis-640204d8-20cc\",\n",
        "      AttentionHeads,\n",
        "      {\"attention\": [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9979304075241089, 0.002069620881229639, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9971864819526672, 0.0010516609763726592, 0.0017618348356336355, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9594857096672058, 0.001310725579969585, 0.03694308176636696, 0.002260456094518304, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9891613125801086, 0.001052629784680903, 0.0048541901633143425, 0.001104357186704874, 0.003827564185485244, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9963598847389221, 0.0007981774397194386, 0.0007764195324853063, 0.00019262291607446969, 0.0002416159404674545, 0.0016313291853293777, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.981286346912384, 0.001863775309175253, 0.006344134453684092, 0.0002960922720376402, 0.004604941233992577, 0.0013966941041871905, 0.00420788861811161, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9950011372566223, 0.0013230532640591264, 0.0007920759962871671, 0.00024644460063427687, 0.0003405151073820889, 0.00016826140927150846, 0.0002837753272615373, 0.0018447580514475703, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9920291900634766, 0.0013539871433749795, 0.0007168339216150343, 9.214139572577551e-05, 0.00013419234892353415, 0.00019706363673321903, 0.00035278062568977475, 0.0002426079590804875, 0.004881155211478472, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5483196377754211, 0.0075838202610611916, 0.28265973925590515, 0.0029685653280466795, 0.14815661311149597, 0.0008172960951924324, 0.0006479909643530846, 0.0017078499076887965, 0.004129356704652309, 0.003009046893566847, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8642559051513672, 0.0036493062507361174, 0.004925842396914959, 0.06957440823316574, 0.038683053106069565, 0.009609837085008621, 0.0008548588375560939, 0.0005643004551529884, 0.003707138355821371, 0.0013952680164948106, 0.002780066104605794, 0.0, 0.0, 0.0, 0.0], [0.30252957344055176, 0.006584456656128168, 0.1401754468679428, 0.030065739527344704, 0.4806697964668274, 0.0005923578282818198, 0.0005309724947437644, 0.0024134721606969833, 0.008505754172801971, 0.0013766245683655143, 0.023764921352267265, 0.0027908117044717073, 0.0, 0.0, 0.0], [0.7773900032043457, 0.003722917754203081, 0.022378109395503998, 0.015139483846724033, 0.017408354207873344, 0.002416662173345685, 0.000709561922121793, 0.0007399603491649032, 0.13850137591362, 0.0023606533650308847, 0.005776867736130953, 0.0028014343697577715, 0.010654616169631481, 0.0, 0.0], [0.9735962152481079, 0.0012812522472813725, 0.002617918187752366, 9.891873924061656e-05, 0.0005098494002595544, 0.00012000725837424397, 0.00045229491661302745, 0.00010779645526781678, 0.002954537281766534, 0.001644388772547245, 0.0012141242623329163, 0.00018637391622178257, 0.0008510759216733277, 0.01436527632176876, 0.0], [0.0990300253033638, 0.000971083587501198, 0.06413238495588303, 0.005246965680271387, 0.8110400438308716, 8.250321843661368e-05, 6.596777529921383e-05, 0.0011991349747404456, 0.0008852760074660182, 0.00024891181965358555, 0.010336406528949738, 7.952339365147054e-05, 0.0049433172680437565, 0.00028918671887367964, 0.0014492359478026628]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9985041618347168, 0.0014959010295569897, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9966639876365662, 0.00046957843005657196, 0.002866449998691678, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9208283424377441, 0.0007845019572414458, 0.07676984369754791, 0.001617342815734446, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9835838675498962, 0.0006018307176418602, 0.0030122774187475443, 0.006188061088323593, 0.0066139851696789265, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9950478076934814, 0.0007798039587214589, 0.0007059435010887682, 0.0003824840241577476, 0.0007798775332048535, 0.0023041421081870794, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9838821887969971, 0.0015907409833744168, 0.0063361418433487415, 0.0004584550915751606, 0.004829846788197756, 0.0009966548532247543, 0.0019060741178691387, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.995481014251709, 0.0010436347220093012, 0.0005037460359744728, 0.0005756175960414112, 0.00046194688184186816, 0.0004267194017302245, 0.0006751787732355297, 0.0008320942288264632, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9798213243484497, 0.008518284186720848, 0.0010190301109105349, 0.00029932931647635996, 0.0003097986045759171, 0.0003330526524223387, 0.001541984616778791, 0.0008126517641358078, 0.007344536483287811, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6489241123199463, 0.008722440339624882, 0.10265976935625076, 0.0038905434776097536, 0.22694909572601318, 0.0011350901331752539, 0.00045881845289841294, 0.0004216691595502198, 0.0053738984279334545, 0.0014645822811871767, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9156036972999573, 0.0013435552828013897, 0.0011673923581838608, 0.04532041400671005, 0.020157871767878532, 0.007576430682092905, 0.0011305580846965313, 0.00024880640557967126, 0.004846310243010521, 0.0007249554619193077, 0.0018800381803885102, 0.0, 0.0, 0.0, 0.0], [0.4962114989757538, 0.0019782893359661102, 0.05844113230705261, 0.00783691368997097, 0.40375715494155884, 0.0005728096584789455, 0.0002963987353723496, 0.0009752871701493859, 0.0046507869847118855, 0.00032523463596589863, 0.02318551018834114, 0.0017689995002001524, 0.0, 0.0, 0.0], [0.8397698402404785, 0.0018636900931596756, 0.005577529780566692, 0.020380759611725807, 0.0292733795940876, 0.004448764491826296, 0.0011503315763548017, 0.00024191653938032687, 0.08715140074491501, 0.0013588638976216316, 0.0026416215114295483, 0.0019266613526269794, 0.004215260501950979, 0.0, 0.0], [0.9744170308113098, 0.0024021149147301912, 0.00455052312463522, 0.00020547708845697343, 0.002662503393366933, 0.00021827162709087133, 0.0008242498734034598, 0.0004054978198837489, 0.002643037121742964, 0.0015235628234222531, 0.0031088448595255613, 7.676492532482371e-05, 0.0018502527382224798, 0.0051117767579853535, 0.0], [0.18470405042171478, 0.00038850627606734633, 0.06518032401800156, 0.0014534658985212445, 0.7318763732910156, 0.00010888298129430041, 2.748575025179889e-05, 0.00011231198004679754, 0.0013672056375071406, 4.0162569348467514e-05, 0.012543701566755772, 5.95860110479407e-05, 0.0007756491540931165, 0.0010575354099273682, 0.00030486442847177386]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9776753783226013, 0.022324683144688606, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9430753588676453, 0.018823042511940002, 0.038101643323898315, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8963208794593811, 0.006171742454171181, 0.07676312327384949, 0.02074429951608181, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8522229790687561, 0.017844438552856445, 0.05478672310709953, 0.012976272031664848, 0.062169551849365234, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9669023752212524, 0.004049547016620636, 0.002062328392639756, 0.007766399998217821, 0.007081815041601658, 0.012137639336287975, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7676379680633545, 0.018502332270145416, 0.045013245195150375, 0.012471744790673256, 0.10591386258602142, 0.018323613330721855, 0.03213733434677124, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.891828179359436, 0.010069617070257664, 0.013090009801089764, 0.010685192421078682, 0.031169477850198746, 0.006502270698547363, 0.02103857509791851, 0.015616719610989094, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9457208514213562, 0.0045005157589912415, 0.013905955478549004, 0.0013248942559584975, 0.01557664293795824, 0.001115700462833047, 0.0007837467710487545, 0.0007244577864184976, 0.016347244381904602, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5131928324699402, 0.006032112520188093, 0.11406763643026352, 0.009519217535853386, 0.30613386631011963, 0.0034422571770846844, 0.002601674757897854, 0.002665397012606263, 0.03412351757287979, 0.008221502415835857, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8395131230354309, 0.008326224982738495, 0.012920893728733063, 0.01282776053994894, 0.029934609308838844, 0.013884111307561398, 0.0034839101135730743, 0.0018691613804548979, 0.039465710520744324, 0.011843967251479626, 0.025930611416697502, 0.0, 0.0, 0.0, 0.0], [0.3585011661052704, 0.004236206877976656, 0.10680253803730011, 0.015598620288074017, 0.41607996821403503, 0.002491925610229373, 0.003585703205317259, 0.0038999069947749376, 0.015854908153414726, 0.005938141606748104, 0.062027692794799805, 0.004983273800462484, 0.0, 0.0, 0.0], [0.612542450428009, 0.004679449368268251, 0.029155218973755836, 0.028240399435162544, 0.08258962631225586, 0.007909866981208324, 0.011129351332783699, 0.0034775957465171814, 0.17027747631072998, 0.010538897477090359, 0.015773506835103035, 0.013295088894665241, 0.01039104349911213, 0.0, 0.0], [0.875810444355011, 0.007089323364198208, 0.01208286453038454, 0.0025996132753789425, 0.021269435063004494, 0.00165357009973377, 0.0019036760786548257, 0.0016310046194121242, 0.02823040634393692, 0.008473552763462067, 0.011080354452133179, 0.002760963048785925, 0.0025940914638340473, 0.02282082475721836, 0.0], [0.5415229201316833, 0.003397508757188916, 0.03888256847858429, 0.007577123120427132, 0.3613935708999634, 0.001381068374030292, 0.0010475177550688386, 0.0012340175453573465, 0.006947703659534454, 0.003757019527256489, 0.0176718570291996, 0.0011743978830054402, 0.0036232525017112494, 0.00219246419146657, 0.008196980692446232]]], \"attentionHeadNames\": [\"L9H9\", \"L9H6\", \"L10H0\"], \"tokens\": [\"<|endoftext|>\", \"When\", \" John\", \" and\", \" Mary\", \" went\", \" to\", \" the\", \" shops\", \",\", \" John\", \" gave\", \" the\", \" bag\", \" to\"]}\n",
        "    )\n",
-       "    </script></div><div style='max-width: 700px;'><h2>Top 3 Negative Logit Attribution Heads</h2><br/><div id=\"circuits-vis-6ae2c4e4-98bd\" style=\"margin: 15px 0;\"/>\n",
+       "    </script></div><div style='max-width: 700px;'><h2>Top 3 Negative Logit Attribution Heads</h2><br/><div id=\"circuits-vis-65209713-881c\" style=\"margin: 15px 0;\"/>\n",
        "    <script crossorigin type=\"module\">\n",
        "    import { render, AttentionHeads } from \"https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js\";\n",
        "    render(\n",
-       "      \"circuits-vis-6ae2c4e4-98bd\",\n",
+       "      \"circuits-vis-65209713-881c\",\n",
        "      AttentionHeads,\n",
        "      {\"attention\": [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9628916382789612, 0.03710832819342613, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9778335094451904, 0.0034864130429923534, 0.018680110573768616, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8737697601318359, 0.005964328069239855, 0.08126737922430038, 0.03899851813912392, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8745138645172119, 0.017276111990213394, 0.018874678760766983, 0.054268334060907364, 0.035066962242126465, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9704693555831909, 0.0035177830141037703, 0.0003239834331907332, 0.004373463336378336, 0.0003953034174628556, 0.02092011272907257, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7694658041000366, 0.006685878615826368, 0.011966485530138016, 0.040966276079416275, 0.03344443440437317, 0.04401002824306488, 0.0934610366821289, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8177722692489624, 0.009112595580518246, 0.0042336746118962765, 0.03900325670838356, 0.012653318233788013, 0.007337852846831083, 0.07410021126270294, 0.0357869453728199, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8309933543205261, 0.018796611577272415, 0.00330327614210546, 0.01983213610947132, 0.004009497817605734, 0.012078307569026947, 0.03246007487177849, 0.010322270914912224, 0.06820447742938995, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.31403300166130066, 0.01522789802402258, 0.19789263606071472, 0.059379592537879944, 0.3236042559146881, 0.009037697687745094, 0.019917670637369156, 0.012475269846618176, 0.013939663767814636, 0.03449229896068573, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7319568991661072, 0.03087025322020054, 0.005064091179519892, 0.08703826367855072, 0.004982746671885252, 0.044725559651851654, 0.02350614219903946, 0.0063977413810789585, 0.0034696452785283327, 0.048653144389390945, 0.013335512951016426, 0.0, 0.0, 0.0, 0.0], [0.13710583746433258, 0.012017901986837387, 0.14592327177524567, 0.015683427453041077, 0.44125232100486755, 0.004342416767030954, 0.011117888614535332, 0.007466601673513651, 0.0012341630645096302, 0.01946023479104042, 0.16842849552631378, 0.035967420786619186, 0.0, 0.0, 0.0], [0.4052484929561615, 0.009720955975353718, 0.03158125653862953, 0.09247038513422012, 0.048808369785547256, 0.008174203336238861, 0.031708307564258575, 0.02787904627621174, 0.14128026366233826, 0.0302340779453516, 0.03392859548330307, 0.03171432018280029, 0.10725171118974686, 0.0, 0.0], [0.808469295501709, 0.0056702690199017525, 0.006755992770195007, 0.008696584962308407, 0.010473440401256084, 0.003464736510068178, 0.006760997697710991, 0.0025889205280691385, 0.015097705647349358, 0.01603602059185505, 0.011288060806691647, 0.017277495935559273, 0.00779919745400548, 0.07962135970592499, 0.0], [0.02939111366868019, 0.0016060526249930263, 0.06005561351776123, 0.008175727911293507, 0.8099164366722107, 0.00039796155760996044, 0.0028282543644309044, 0.0033632616978138685, 0.0001569116284372285, 0.0036459483671933413, 0.05494074895977974, 0.001666039228439331, 0.015368940308690071, 5.016689101466909e-05, 0.008436810225248337]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9124024510383606, 0.08759760111570358, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.97407466173172, 0.01434413529932499, 0.011581187136471272, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8240247964859009, 0.03700513392686844, 0.06643965095281601, 0.07253041118383408, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5370795726776123, 0.15483324229717255, 0.0903128981590271, 0.12131187319755554, 0.0964624434709549, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9388002157211304, 0.028733234852552414, 0.0003431019140407443, 0.010481251403689384, 0.000839930260553956, 0.02080223336815834, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6318807005882263, 0.059302832931280136, 0.03517911583185196, 0.0317409411072731, 0.1174689382314682, 0.06869616359472275, 0.05573126673698425, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7625216841697693, 0.04435169696807861, 0.010552966967225075, 0.034276288002729416, 0.014203979633748531, 0.012703591957688332, 0.07259687036275864, 0.048792947083711624, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5725991725921631, 0.12681783735752106, 0.03727060556411743, 0.0431414395570755, 0.04129580408334732, 0.046447426080703735, 0.029164383187890053, 0.03751189261674881, 0.06575141847133636, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.12316936254501343, 0.026347072795033455, 0.27960509061813354, 0.041989635676145554, 0.44105827808380127, 0.006138973869383335, 0.007793003693223, 0.007175281643867493, 0.011286810971796513, 0.055436521768569946, 0.0, 0.0, 0.0, 0.0, 0.0], [0.44855940341949463, 0.1432485729455948, 0.010774216614663601, 0.07855041325092316, 0.01404787041246891, 0.014864671975374222, 0.008633698336780071, 0.0030146560166031122, 0.012792127206921577, 0.16408587992191315, 0.10142850130796432, 0.0, 0.0, 0.0, 0.0], [0.10228201746940613, 0.0238832775503397, 0.055250268429517746, 0.0520680733025074, 0.5759841799736023, 0.0011948152678087354, 0.0013662497512996197, 0.0025677781086415052, 0.0018699930515140295, 0.043160032480955124, 0.10887196660041809, 0.03150142729282379, 0.0, 0.0, 0.0], [0.39313194155693054, 0.04406267777085304, 0.04009911045432091, 0.07407406717538834, 0.04061643034219742, 0.007346798665821552, 0.009507199749350548, 0.013865168206393719, 0.0640539675951004, 0.04894229769706726, 0.0627753809094429, 0.0999651551246643, 0.10155977308750153, 0.0, 0.0], [0.7573198676109314, 0.015095721930265427, 0.007350177504122257, 0.013156400062143803, 0.005726401228457689, 0.008556416258215904, 0.008282607421278954, 0.010307075455784798, 0.002904881490394473, 0.017329057678580284, 0.015188980847597122, 0.0796574279665947, 0.030244464054703712, 0.02888045459985733, 0.0], [0.08579524606466293, 0.023773644119501114, 0.028759371489286423, 0.050303805619478226, 0.674644947052002, 0.0010763936443254352, 0.0011514866491779685, 0.002721975091844797, 0.0005617713322862983, 0.039091579616069794, 0.04861969128251076, 0.015520088374614716, 0.0040845321491360664, 0.00043291584006510675, 0.023462524637579918]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9691730737686157, 0.03082684986293316, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9864587187767029, 0.0076598357409238815, 0.00588154187425971, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9318376183509827, 0.011915593408048153, 0.02053167298436165, 0.03571505844593048, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8786704540252686, 0.03427596017718315, 0.01315787062048912, 0.02705124393105507, 0.04684450104832649, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9850097298622131, 0.00501946872100234, 0.0013920071069151163, 0.0018760499078780413, 0.004171107430011034, 0.002531560370698571, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8907089829444885, 0.02026253007352352, 0.020904401317238808, 0.009134626016020775, 0.04057806357741356, 0.003215113654732704, 0.015196278691291809, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9318532347679138, 0.012423796579241753, 0.007052858360111713, 0.00626762630417943, 0.009715708903968334, 0.0013882736675441265, 0.012697475962340832, 0.018601125106215477, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7722940444946289, 0.03829518333077431, 0.03943629562854767, 0.00972924567759037, 0.11019386351108551, 0.009773456491529942, 0.00886810664087534, 0.003977742046117783, 0.007432003039866686, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6659272909164429, 0.019893741235136986, 0.05748209357261658, 0.026046503335237503, 0.1897869110107422, 0.001734035206027329, 0.004116130527108908, 0.004545124247670174, 0.007496052421629429, 0.022972112521529198, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8588331341743469, 0.01852157525718212, 0.010080324485898018, 0.0161746833473444, 0.044801417738199234, 0.018235526978969574, 0.0026932875625789165, 0.0009044647449627519, 0.004338268656283617, 0.01020037941634655, 0.015216901898384094, 0.0, 0.0, 0.0, 0.0], [0.7452806234359741, 0.02059219218790531, 0.033448558300733566, 0.04560310021042824, 0.09397682547569275, 0.0019108005799353123, 0.0039161862805485725, 0.004672198090702295, 0.003216084325686097, 0.0180679801851511, 0.0163005068898201, 0.013014941476285458, 0.0, 0.0, 0.0], [0.7450795769691467, 0.016002941876649857, 0.02564326673746109, 0.03000519424676895, 0.06691146641969681, 0.0032243167515844107, 0.006530732847750187, 0.006865624338388443, 0.04355334863066673, 0.012662945315241814, 0.010547308251261711, 0.010315663181245327, 0.022657567635178566, 0.0, 0.0], [0.8363587260246277, 0.013512525707483292, 0.01655869372189045, 0.008880337700247765, 0.05630703642964363, 0.0022557524498552084, 0.0023984003346413374, 0.0012933476828038692, 0.003150224220007658, 0.013540910556912422, 0.02165023609995842, 0.005392791703343391, 0.00432277237996459, 0.014378219842910767, 0.0], [0.7888551354408264, 0.014087573625147343, 0.022160783410072327, 0.04220619052648544, 0.06532405316829681, 0.0011144893942400813, 0.001956802560016513, 0.002514239400625229, 0.001448699738830328, 0.012026125565171242, 0.009607555344700813, 0.006895523052662611, 0.004557081963866949, 0.003263341262936592, 0.023982450366020203]]], \"attentionHeadNames\": [\"L10H7\", \"L11H10\", \"L11H2\"], \"tokens\": [\"<|endoftext|>\", \"When\", \" John\", \" and\", \" Mary\", \" went\", \" to\", \" the\", \" shops\", \",\", \" John\", \" gave\", \" the\", \" bag\", \" to\"]}\n",
        "    )\n",
@@ -4414,10 +4414,10 @@
    "execution_count": 19,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:37:19.457595Z",
-     "iopub.status.busy": "2024-09-05T17:37:19.457194Z",
-     "iopub.status.idle": "2024-09-05T17:37:19.740830Z",
-     "shell.execute_reply": "2024-09-05T17:37:19.740228Z"
+     "iopub.execute_input": "2024-09-10T17:11:01.618130Z",
+     "iopub.status.busy": "2024-09-10T17:11:01.617964Z",
+     "iopub.status.idle": "2024-09-10T17:11:01.904076Z",
+     "shell.execute_reply": "2024-09-10T17:11:01.903454Z"
     }
    },
    "outputs": [
@@ -4449,10 +4449,10 @@
    "execution_count": 20,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:37:19.743228Z",
-     "iopub.status.busy": "2024-09-05T17:37:19.742845Z",
-     "iopub.status.idle": "2024-09-05T17:37:19.747309Z",
-     "shell.execute_reply": "2024-09-05T17:37:19.746759Z"
+     "iopub.execute_input": "2024-09-10T17:11:01.906447Z",
+     "iopub.status.busy": "2024-09-10T17:11:01.906108Z",
+     "iopub.status.idle": "2024-09-10T17:11:01.910654Z",
+     "shell.execute_reply": "2024-09-10T17:11:01.910072Z"
     }
    },
    "outputs": [
@@ -4493,10 +4493,10 @@
    "execution_count": 21,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:37:19.749382Z",
-     "iopub.status.busy": "2024-09-05T17:37:19.749051Z",
-     "iopub.status.idle": "2024-09-05T17:38:07.638162Z",
-     "shell.execute_reply": "2024-09-05T17:38:07.637577Z"
+     "iopub.execute_input": "2024-09-10T17:11:01.912731Z",
+     "iopub.status.busy": "2024-09-10T17:11:01.912420Z",
+     "iopub.status.idle": "2024-09-10T17:11:49.025260Z",
+     "shell.execute_reply": "2024-09-10T17:11:49.024580Z"
     }
    },
    "outputs": [],
@@ -4554,10 +4554,10 @@
    "execution_count": 22,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:38:07.640830Z",
-     "iopub.status.busy": "2024-09-05T17:38:07.640643Z",
-     "iopub.status.idle": "2024-09-05T17:38:07.672651Z",
-     "shell.execute_reply": "2024-09-05T17:38:07.672038Z"
+     "iopub.execute_input": "2024-09-10T17:11:49.028139Z",
+     "iopub.status.busy": "2024-09-10T17:11:49.027746Z",
+     "iopub.status.idle": "2024-09-10T17:11:49.059638Z",
+     "shell.execute_reply": "2024-09-10T17:11:49.059074Z"
     }
    },
    "outputs": [
@@ -5696,9 +5696,9 @@
        }
       },
       "text/html": [
-       "<div>                            <div id=\"6767b2c2-3357-400a-bf47-8577e0e58602\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"6767b2c2-3357-400a-bf47-8577e0e58602\")) {                    Plotly.newPlot(                        \"6767b2c2-3357-400a-bf47-8577e0e58602\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"x\":[\"\\u003c|endoftext|\\u003e_0\",\"When_1\",\" John_2\",\" and_3\",\" Mary_4\",\" went_5\",\" to_6\",\" the_7\",\" shops_8\",\",_9\",\" John_10\",\" gave_11\",\" the_12\",\" bag_13\",\" to_14\"],\"z\":[[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.00065016746521,-0.00024725322145968676,9.061812306754291e-06,-0.00036435198853723705,-4.832966806134209e-05],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0010517835617065,-2.6816253011929803e-05,-2.0540108380373567e-05,-0.0004592325130943209,-0.0005939850234426558],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0002665519714355,0.0008679538150317967,0.0005159862921573222,-0.0009933760156854987,-0.0008652352844364941],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9949080944061279,0.005429603159427643,0.0016055518062785268,-0.0006179149495437741,-0.0016324687749147415],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9675664305686951,0.03134222328662872,0.0028418514411896467,-0.0012303927214816213,-0.0009862943552434444],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9675208926200867,0.031000729650259018,0.001782458508387208,-0.0004856795712839812,-0.000646778498776257],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9228320121765137,0.05134553834795952,0.004729225765913725,0.0009345413418486714,0.017047081142663956],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.6565485000610352,0.02385673113167286,0.002357447287067771,-1.7318130630883388e-05,0.3186914026737213],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.027302434667944908,0.03142485395073891,0.0018206859240308404,0.0007993190083652735,0.9383869171142578],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.026841893792152405,0.020981015637516975,0.0012513356050476432,0.0003238087520003319,1.0048280954360962],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.005687932018190622,0.014263695105910301,0.0004871227720286697,-8.984619489638135e-05,0.9914218783378601]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Position: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Position\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Logit Difference From Patched Residual Stream\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "<div>                            <div id=\"ad844fd5-560b-4c3a-bc40-4b48c6c877b8\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"ad844fd5-560b-4c3a-bc40-4b48c6c877b8\")) {                    Plotly.newPlot(                        \"ad844fd5-560b-4c3a-bc40-4b48c6c877b8\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"x\":[\"\\u003c|endoftext|\\u003e_0\",\"When_1\",\" John_2\",\" and_3\",\" Mary_4\",\" went_5\",\" to_6\",\" the_7\",\" shops_8\",\",_9\",\" John_10\",\" gave_11\",\" the_12\",\" bag_13\",\" to_14\"],\"z\":[[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.00065016746521,-0.00024725322145968676,9.061812306754291e-06,-0.00036435198853723705,-4.832966806134209e-05],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0010517835617065,-2.6816253011929803e-05,-2.0540108380373567e-05,-0.0004592325130943209,-0.0005939850234426558],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0002665519714355,0.0008679538150317967,0.0005159862921573222,-0.0009933760156854987,-0.0008652352844364941],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9949080944061279,0.005429603159427643,0.0016055518062785268,-0.0006179149495437741,-0.0016324687749147415],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9675664305686951,0.03134222328662872,0.0028418514411896467,-0.0012303927214816213,-0.0009862943552434444],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9675208926200867,0.031000729650259018,0.001782458508387208,-0.0004856795712839812,-0.000646778498776257],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9228320121765137,0.05134553834795952,0.004729225765913725,0.0009345413418486714,0.017047081142663956],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.6565485000610352,0.02385673113167286,0.002357447287067771,-1.7318130630883388e-05,0.3186914026737213],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.027302434667944908,0.03142485395073891,0.0018206859240308404,0.0007993190083652735,0.9383869171142578],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.026841893792152405,0.020981015637516975,0.0012513356050476432,0.0003238087520003319,1.0048280954360962],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.005687932018190622,0.014263695105910301,0.0004871227720286697,-8.984619489638135e-05,0.9914218783378601]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Position: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Position\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Logit Difference From Patched Residual Stream\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('6767b2c2-3357-400a-bf47-8577e0e58602');\n",
+       "var gd = document.getElementById('ad844fd5-560b-4c3a-bc40-4b48c6c877b8');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -5758,10 +5758,10 @@
    "execution_count": 23,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:38:07.674750Z",
-     "iopub.status.busy": "2024-09-05T17:38:07.674574Z",
-     "iopub.status.idle": "2024-09-05T17:39:43.389417Z",
-     "shell.execute_reply": "2024-09-05T17:39:43.388763Z"
+     "iopub.execute_input": "2024-09-10T17:11:49.061736Z",
+     "iopub.status.busy": "2024-09-10T17:11:49.061555Z",
+     "iopub.status.idle": "2024-09-10T17:13:23.322359Z",
+     "shell.execute_reply": "2024-09-10T17:13:23.321778Z"
     }
    },
    "outputs": [],
@@ -5812,10 +5812,10 @@
    "execution_count": 24,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:39:43.392123Z",
-     "iopub.status.busy": "2024-09-05T17:39:43.391939Z",
-     "iopub.status.idle": "2024-09-05T17:39:43.422475Z",
-     "shell.execute_reply": "2024-09-05T17:39:43.421905Z"
+     "iopub.execute_input": "2024-09-10T17:13:23.325050Z",
+     "iopub.status.busy": "2024-09-10T17:13:23.324709Z",
+     "iopub.status.idle": "2024-09-10T17:13:23.356150Z",
+     "shell.execute_reply": "2024-09-10T17:13:23.355521Z"
     }
    },
    "outputs": [
@@ -6954,9 +6954,9 @@
        }
       },
       "text/html": [
-       "<div>                            <div id=\"f4f11dd9-871d-4910-93fd-3680b4af3721\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"f4f11dd9-871d-4910-93fd-3680b4af3721\")) {                    Plotly.newPlot(                        \"f4f11dd9-871d-4910-93fd-3680b4af3721\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"x\":[\"\\u003c|endoftext|\\u003e_0\",\"When_1\",\" John_2\",\" and_3\",\" Mary_4\",\" went_5\",\" to_6\",\" the_7\",\" shops_8\",\",_9\",\" John_10\",\" gave_11\",\" the_12\",\" bag_13\",\" to_14\"],\"z\":[[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.03545692563056946,-0.00024725322145968676,9.061812306754291e-06,-0.00036435198853723705,-4.832966806134209e-05],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.0029837191104888916,7.950901635922492e-05,2.097641845466569e-05,8.054944191826507e-05,-0.0005966364406049252],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.0019127808045595884,0.000666748033836484,0.00039496077806688845,-0.0007048076367937028,-0.0002728612453211099],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.15463140606880188,0.0038022694643586874,0.0005171945667825639,-0.00012015292304567993,-0.0005605234182439744],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.005405941512435675,0.019581804051995277,0.0010072707664221525,-0.00024272232258226722,0.0007941504009068012],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.35209786891937256,0.0010532510932534933,0.00022453156998381019,0.00013320863945409656,8.259674359578639e-05],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.11985944956541061,0.02124394290149212,0.002728075487539172,0.0013410811079666018,0.017973870038986206],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.013310795649886131,0.011509508825838566,0.00037489054375328124,-4.121446545468643e-05,0.2976022958755493],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.001500166254118085,0.017352566123008728,0.0005847889697179198,0.0010113989701494575,0.5697317123413086],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.0001285099278902635,0.006301181390881538,0.00014156564429868013,0.00031266608857549727,0.2715233862400055],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.0009370585321448743,8.679202437633649e-05,0.00033203151542693377,1.543864300401765e-06,-0.1929759979248047],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.4061770737171173]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Position: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Position\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Logit Difference From Patched Attention Layer\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "<div>                            <div id=\"15943854-8114-4fd6-addb-4e62d14ddf33\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"15943854-8114-4fd6-addb-4e62d14ddf33\")) {                    Plotly.newPlot(                        \"15943854-8114-4fd6-addb-4e62d14ddf33\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"x\":[\"\\u003c|endoftext|\\u003e_0\",\"When_1\",\" John_2\",\" and_3\",\" Mary_4\",\" went_5\",\" to_6\",\" the_7\",\" shops_8\",\",_9\",\" John_10\",\" gave_11\",\" the_12\",\" bag_13\",\" to_14\"],\"z\":[[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.03545692563056946,-0.00024725322145968676,9.061812306754291e-06,-0.00036435198853723705,-4.832966806134209e-05],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.0029837191104888916,7.950901635922492e-05,2.097641845466569e-05,8.054944191826507e-05,-0.0005966364406049252],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.0019127808045595884,0.000666748033836484,0.00039496077806688845,-0.0007048076367937028,-0.0002728612453211099],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.15463140606880188,0.0038022694643586874,0.0005171945667825639,-0.00012015292304567993,-0.0005605234182439744],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.005405941512435675,0.019581804051995277,0.0010072707664221525,-0.00024272232258226722,0.0007941504009068012],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.35209786891937256,0.0010532510932534933,0.00022453156998381019,0.00013320863945409656,8.259674359578639e-05],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.11985944956541061,0.02124394290149212,0.002728075487539172,0.0013410811079666018,0.017973870038986206],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.013310795649886131,0.011509508825838566,0.00037489054375328124,-4.121446545468643e-05,0.2976022958755493],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.001500166254118085,0.017352566123008728,0.0005847889697179198,0.0010113989701494575,0.5697317123413086],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.0001285099278902635,0.006301181390881538,0.00014156564429868013,0.00031266608857549727,0.2715233862400055],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.0009370585321448743,8.679202437633649e-05,0.00033203151542693377,1.543864300401765e-06,-0.1929759979248047],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.4061770737171173]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Position: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Position\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Logit Difference From Patched Attention Layer\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('f4f11dd9-871d-4910-93fd-3680b4af3721');\n",
+       "var gd = document.getElementById('15943854-8114-4fd6-addb-4e62d14ddf33');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -7018,10 +7018,10 @@
    "execution_count": 25,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:39:43.424779Z",
-     "iopub.status.busy": "2024-09-05T17:39:43.424599Z",
-     "iopub.status.idle": "2024-09-05T17:39:43.453834Z",
-     "shell.execute_reply": "2024-09-05T17:39:43.453284Z"
+     "iopub.execute_input": "2024-09-10T17:13:23.358161Z",
+     "iopub.status.busy": "2024-09-10T17:13:23.357988Z",
+     "iopub.status.idle": "2024-09-10T17:13:23.387717Z",
+     "shell.execute_reply": "2024-09-10T17:13:23.387108Z"
     }
    },
    "outputs": [
@@ -8160,9 +8160,9 @@
        }
       },
       "text/html": [
-       "<div>                            <div id=\"fa8585f3-ce45-40a0-b698-398c00aca11d\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"fa8585f3-ce45-40a0-b698-398c00aca11d\")) {                    Plotly.newPlot(                        \"fa8585f3-ce45-40a0-b698-398c00aca11d\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"x\":[\"\\u003c|endoftext|\\u003e_0\",\"When_1\",\" John_2\",\" and_3\",\" Mary_4\",\" went_5\",\" to_6\",\" the_7\",\" shops_8\",\",_9\",\" John_10\",\" gave_11\",\" the_12\",\" bag_13\",\" to_14\"],\"z\":[[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.8507904410362244,-0.00027849970501847565,-7.350136729655787e-05,-0.00047316084965132177,3.9704162190901116e-05],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.008864667266607285,0.00022147741401568055,0.00014985552115831524,-4.8866662837099284e-05,0.000304174842312932],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.013550027273595333,5.839834557264112e-05,-0.0003300177922938019,-0.0006387235480360687,0.0007728383643552661],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0019474170403555036,0.0004988359869457781,0.00017297992599196732,0.00016838189912959933,0.0004079157952219248],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.019786769524216652,0.004128897562623024,-4.849747710977681e-05,-0.00016989219875540584,0.0007919353083707392],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.09652478247880936,-0.001882205600850284,-0.00048380010412074625,0.000710244697984308,-0.00018361916590947658],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.015899553894996643,-0.0008504678844474256,0.00012357627565506846,2.8863551051472314e-05,-0.007237736601382494],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.01036097388714552,0.003150892909616232,0.0005310557899065316,0.00023543930728919804,0.008496489375829697],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.012532956898212433,2.2285346858552657e-05,-0.0003539476892910898,8.638927829451859e-05,-0.02163148671388626],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.00033434730721637607,0.0008097903919406235,1.6344823961844668e-05,0.00012938254803884774,0.031624484807252884],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0013600102392956614,-0.0001945940311998129,-9.914294059854001e-05,-0.00014133071817923337,0.02876460924744606],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.020449254661798477]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Position: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Position\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Logit Difference From Patched MLP Layer\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "<div>                            <div id=\"4a8f65af-b38f-42b0-96a4-9ceb6ed2fec9\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"4a8f65af-b38f-42b0-96a4-9ceb6ed2fec9\")) {                    Plotly.newPlot(                        \"4a8f65af-b38f-42b0-96a4-9ceb6ed2fec9\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"x\":[\"\\u003c|endoftext|\\u003e_0\",\"When_1\",\" John_2\",\" and_3\",\" Mary_4\",\" went_5\",\" to_6\",\" the_7\",\" shops_8\",\",_9\",\" John_10\",\" gave_11\",\" the_12\",\" bag_13\",\" to_14\"],\"z\":[[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.8507904410362244,-0.00027849970501847565,-7.350136729655787e-05,-0.00047316084965132177,3.9704162190901116e-05],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.008864667266607285,0.00022147741401568055,0.00014985552115831524,-4.8866662837099284e-05,0.000304174842312932],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.013550027273595333,5.839834557264112e-05,-0.0003300177922938019,-0.0006387235480360687,0.0007728383643552661],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0019474170403555036,0.0004988359869457781,0.00017297992599196732,0.00016838189912959933,0.0004079157952219248],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.019786769524216652,0.004128897562623024,-4.849747710977681e-05,-0.00016989219875540584,0.0007919353083707392],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.09652478247880936,-0.001882205600850284,-0.00048380010412074625,0.000710244697984308,-0.00018361916590947658],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.015899553894996643,-0.0008504678844474256,0.00012357627565506846,2.8863551051472314e-05,-0.007237736601382494],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.01036097388714552,0.003150892909616232,0.0005310557899065316,0.00023543930728919804,0.008496489375829697],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.012532956898212433,2.2285346858552657e-05,-0.0003539476892910898,8.638927829451859e-05,-0.02163148671388626],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.00033434730721637607,0.0008097903919406235,1.6344823961844668e-05,0.00012938254803884774,0.031624484807252884],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0013600102392956614,-0.0001945940311998129,-9.914294059854001e-05,-0.00014133071817923337,0.02876460924744606],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.020449254661798477]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Position: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Position\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Logit Difference From Patched MLP Layer\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('fa8585f3-ce45-40a0-b698-398c00aca11d');\n",
+       "var gd = document.getElementById('4a8f65af-b38f-42b0-96a4-9ceb6ed2fec9');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -8216,10 +8216,10 @@
    "execution_count": 26,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:39:43.456213Z",
-     "iopub.status.busy": "2024-09-05T17:39:43.455797Z",
-     "iopub.status.idle": "2024-09-05T17:40:21.642918Z",
-     "shell.execute_reply": "2024-09-05T17:40:21.642253Z"
+     "iopub.execute_input": "2024-09-10T17:13:23.389684Z",
+     "iopub.status.busy": "2024-09-10T17:13:23.389513Z",
+     "iopub.status.idle": "2024-09-10T17:14:01.169166Z",
+     "shell.execute_reply": "2024-09-10T17:14:01.168476Z"
     }
    },
    "outputs": [],
@@ -8266,10 +8266,10 @@
    "execution_count": 27,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:40:21.645690Z",
-     "iopub.status.busy": "2024-09-05T17:40:21.645269Z",
-     "iopub.status.idle": "2024-09-05T17:40:21.676102Z",
-     "shell.execute_reply": "2024-09-05T17:40:21.675514Z"
+     "iopub.execute_input": "2024-09-10T17:14:01.171896Z",
+     "iopub.status.busy": "2024-09-10T17:14:01.171713Z",
+     "iopub.status.idle": "2024-09-10T17:14:01.202914Z",
+     "shell.execute_reply": "2024-09-10T17:14:01.202306Z"
     }
    },
    "outputs": [
@@ -9355,9 +9355,9 @@
        }
       },
       "text/html": [
-       "<div>                            <div id=\"a9508ddb-9c9f-4520-a179-20bc68dcb955\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"a9508ddb-9c9f-4520-a179-20bc68dcb955\")) {                    Plotly.newPlot(                        \"a9508ddb-9c9f-4520-a179-20bc68dcb955\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.000949342327658087,0.016124119982123375,0.0018556242575868964,0.0034389242064207792,-0.009823138825595379,0.011059673503041267,-0.0040638200007379055,-0.0015785341383889318,-0.001208073808811605,0.003828749991953373,-0.004257441032677889,-0.0011428623693063855],[-0.001076644053682685,-0.0003779782564379275,3.0206042538338806e-06,-0.000260510336374864,-0.00014149851631373167,0.0038328110240399837,-0.0004281202855054289,-0.0014291484840214252,-0.0009217541082762182,0.0006938663427717984,0.00043359093251638114,-0.0035708239302039146],[-0.0004966544220224023,0.0008070718613453209,0.0005420306115411222,-0.0005315592279657722,-0.0007152119651436806,-0.0010380810126662254,-0.0009492752142250538,-8.538240945199504e-05,0.0002774928288999945,0.0021076432894915342,-0.0001976146304514259,-0.0016398524167016149],[0.11626167595386505,0.00025017315056174994,-0.0014669395750388503,-0.0003969745012000203,0.01896214485168457,-0.00018882131553255022,0.011170395649969578,-0.0013304082676768303,-0.0007363561890088022,-0.00030310085276141763,-0.00014579448907170445,-0.0002225178322987631],[-0.0016502231592312455,0.00029299859306775033,-0.0014352232683449984,0.030841777101159096,-0.007431894540786743,-0.0002820908557623625,0.006017110776156187,-0.011006947606801987,-0.0012651296565309167,0.001490030437707901,-0.00018016225658357143,0.0029450219590216875],[-0.004210856277495623,0.0029608632903546095,0.002045620232820511,0.0013410139363259077,-0.0012189144035801291,0.34349939227104187,0.0005627049831673503,-0.0001257913827430457,-0.005152345169335604,0.016241183504462242,0.017089638859033585,-0.004173669498413801],[0.039775215089321136,0.015227536670863628,-0.001023011514917016,0.0008081123232841492,-0.004935599863529205,-0.002123551908880472,-0.014273260720074177,0.0013749118661507964,0.0014838549541309476,0.1302708089351654,-0.00033538774005137384,0.0012914760736748576],[0.0003724069392774254,0.0195147804915905,0.00022308839834295213,0.12424899637699127,-0.0004041232750751078,-0.007652163505554199,0.0013012762647122145,-0.0011244030902162194,-0.007449581753462553,0.19224178791046143,-0.0032753082923591137,-0.000501957256346941],[-0.001007740618661046,3.1212908652378246e-05,-0.0008586570620536804,0.012360110878944397,-0.00040429108776152134,-0.004328995477408171,0.31855517625808716,0.002329523442313075,0.002118081087246537,0.0001410957775078714,0.27793484926223755,0.005738543812185526],[0.0058897752314805984,-0.000968372100032866,0.009125681594014168,0.020675498992204666,-0.03700448200106621,0.014263628050684929,-0.04828385263681412,0.05834108963608742,0.0006516450084745884,0.2636081576347351,0.0004918886115774512,-0.0026110773906111717],[0.08374276012182236,0.020676370710134506,-0.0037437702994793653,0.01085111778229475,-0.0010960765648633242,0.0004738992138300091,0.04817944020032883,-0.47990989685058594,0.00018412260396871716,0.011861777864396572,0.06088537722826004,0.0008459034143015742],[0.005328110884875059,-0.011493667028844357,-0.11350765824317932,0.006330112461000681,0.0003166264505125582,-0.0011600126745179296,-0.022669030353426933,0.004070901777595282,0.007316238712519407,-0.008345123380422592,-0.27817434072494507,0.003635196480900049]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Logit Difference From Patched Head Output\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "<div>                            <div id=\"1555ca46-05e4-4d5a-8182-16f1ca23918a\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"1555ca46-05e4-4d5a-8182-16f1ca23918a\")) {                    Plotly.newPlot(                        \"1555ca46-05e4-4d5a-8182-16f1ca23918a\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.000949342327658087,0.016124119982123375,0.0018556242575868964,0.0034389242064207792,-0.009823138825595379,0.011059673503041267,-0.0040638200007379055,-0.0015785341383889318,-0.001208073808811605,0.003828749991953373,-0.004257441032677889,-0.0011428623693063855],[-0.001076644053682685,-0.0003779782564379275,3.0206042538338806e-06,-0.000260510336374864,-0.00014149851631373167,0.0038328110240399837,-0.0004281202855054289,-0.0014291484840214252,-0.0009217541082762182,0.0006938663427717984,0.00043359093251638114,-0.0035708239302039146],[-0.0004966544220224023,0.0008070718613453209,0.0005420306115411222,-0.0005315592279657722,-0.0007152119651436806,-0.0010380810126662254,-0.0009492752142250538,-8.538240945199504e-05,0.0002774928288999945,0.0021076432894915342,-0.0001976146304514259,-0.0016398524167016149],[0.11626167595386505,0.00025017315056174994,-0.0014669395750388503,-0.0003969745012000203,0.01896214485168457,-0.00018882131553255022,0.011170395649969578,-0.0013304082676768303,-0.0007363561890088022,-0.00030310085276141763,-0.00014579448907170445,-0.0002225178322987631],[-0.0016502231592312455,0.00029299859306775033,-0.0014352232683449984,0.030841777101159096,-0.007431894540786743,-0.0002820908557623625,0.006017110776156187,-0.011006947606801987,-0.0012651296565309167,0.001490030437707901,-0.00018016225658357143,0.0029450219590216875],[-0.004210856277495623,0.0029608632903546095,0.002045620232820511,0.0013410139363259077,-0.0012189144035801291,0.34349939227104187,0.0005627049831673503,-0.0001257913827430457,-0.005152345169335604,0.016241183504462242,0.017089638859033585,-0.004173669498413801],[0.039775215089321136,0.015227536670863628,-0.001023011514917016,0.0008081123232841492,-0.004935599863529205,-0.002123551908880472,-0.014273260720074177,0.0013749118661507964,0.0014838549541309476,0.1302708089351654,-0.00033538774005137384,0.0012914760736748576],[0.0003724069392774254,0.0195147804915905,0.00022308839834295213,0.12424899637699127,-0.0004041232750751078,-0.007652163505554199,0.0013012762647122145,-0.0011244030902162194,-0.007449581753462553,0.19224178791046143,-0.0032753082923591137,-0.000501957256346941],[-0.001007740618661046,3.1212908652378246e-05,-0.0008586570620536804,0.012360110878944397,-0.00040429108776152134,-0.004328995477408171,0.31855517625808716,0.002329523442313075,0.002118081087246537,0.0001410957775078714,0.27793484926223755,0.005738543812185526],[0.0058897752314805984,-0.000968372100032866,0.009125681594014168,0.020675498992204666,-0.03700448200106621,0.014263628050684929,-0.04828385263681412,0.05834108963608742,0.0006516450084745884,0.2636081576347351,0.0004918886115774512,-0.0026110773906111717],[0.08374276012182236,0.020676370710134506,-0.0037437702994793653,0.01085111778229475,-0.0010960765648633242,0.0004738992138300091,0.04817944020032883,-0.47990989685058594,0.00018412260396871716,0.011861777864396572,0.06088537722826004,0.0008459034143015742],[0.005328110884875059,-0.011493667028844357,-0.11350765824317932,0.006330112461000681,0.0003166264505125582,-0.0011600126745179296,-0.022669030353426933,0.004070901777595282,0.007316238712519407,-0.008345123380422592,-0.27817434072494507,0.003635196480900049]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Logit Difference From Patched Head Output\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('a9508ddb-9c9f-4520-a179-20bc68dcb955');\n",
+       "var gd = document.getElementById('1555ca46-05e4-4d5a-8182-16f1ca23918a');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -9420,10 +9420,10 @@
    "execution_count": 28,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:40:21.678407Z",
-     "iopub.status.busy": "2024-09-05T17:40:21.678072Z",
-     "iopub.status.idle": "2024-09-05T17:40:59.792010Z",
-     "shell.execute_reply": "2024-09-05T17:40:59.791359Z"
+     "iopub.execute_input": "2024-09-10T17:14:01.205307Z",
+     "iopub.status.busy": "2024-09-10T17:14:01.204963Z",
+     "iopub.status.idle": "2024-09-10T17:14:38.905657Z",
+     "shell.execute_reply": "2024-09-10T17:14:38.905010Z"
     }
    },
    "outputs": [],
@@ -9458,10 +9458,10 @@
    "execution_count": 29,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:40:59.794896Z",
-     "iopub.status.busy": "2024-09-05T17:40:59.794676Z",
-     "iopub.status.idle": "2024-09-05T17:40:59.825179Z",
-     "shell.execute_reply": "2024-09-05T17:40:59.824628Z"
+     "iopub.execute_input": "2024-09-10T17:14:38.908448Z",
+     "iopub.status.busy": "2024-09-10T17:14:38.908064Z",
+     "iopub.status.idle": "2024-09-10T17:14:38.938953Z",
+     "shell.execute_reply": "2024-09-10T17:14:38.938403Z"
     }
    },
    "outputs": [
@@ -10547,9 +10547,9 @@
        }
       },
       "text/html": [
-       "<div>                            <div id=\"d15c92bc-9a5e-459f-a673-8ffed2367454\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"d15c92bc-9a5e-459f-a673-8ffed2367454\")) {                    Plotly.newPlot(                        \"d15c92bc-9a5e-459f-a673-8ffed2367454\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[-0.00019808451179414988,0.005339857656508684,0.0006520813331007957,0.0035054110921919346,-0.008983243256807327,0.003481481224298477,-0.0008612077799625695,-3.356226807227358e-05,0.0005169931682758033,0.000442283577285707,-0.003907017409801483,-0.0001884857047116384],[-0.0004399341996759176,-0.00044604254071600735,-6.719166412949562e-05,7.296437252080068e-05,-3.587806349969469e-05,-0.001932314014993608,-0.0001571049797348678,1.550576780573465e-05,0.00020597163529600948,0.0003376699751242995,0.00035139694227837026,-0.0005663632764481008],[0.0002103011793224141,-0.0007198435487225652,0.00048675358993932605,-0.0005971734644845128,-0.0005919041577726603,-0.0005438430234789848,-0.0002275186125189066,-0.00047960481606423855,0.00020617301925085485,0.0011842446401715279,-0.00035743816988542676,-0.0009093025582842529],[0.0010397254955023527,-0.0001205221051350236,-7.69247199059464e-05,-0.0007269251509569585,-0.001310002407990396,-0.002310896525159478,0.010987145826220512,-5.06454634887632e-05,0.00014364650996867567,0.00015129870735108852,-7.920695497887209e-05,-1.9667490050778724e-05],[-0.0005367277772165835,-0.0008130795322358608,-0.000133275767439045,0.030610064044594765,-0.007185581140220165,0.00014941922563593835,0.001333831693045795,-0.011422313749790192,-0.0005329688428901136,0.0005127307958900928,0.00037368229823186994,0.002956030424684286],[7.4508234320092015e-06,6.712453796353657e-06,0.0015982352197170258,0.00033918028930202127,-0.0012586521916091442,-5.423662514658645e-05,0.0006328836898319423,-0.0002699077595025301,7.430685946019366e-05,-0.006704063154757023,0.0031767694745212793,-0.001728456816636026],[0.04863159358501434,0.015315738506615162,-0.0004640990518964827,-0.0001160583269665949,-4.839679240831174e-05,-0.003953501116484404,-0.017375119030475616,-0.0001535809424240142,0.0012203240767121315,-0.00017975950322579592,-0.0004269791825208813,0.0001233748917002231],[-2.849436532414984e-05,-0.001384309260174632,-0.00012176390737295151,0.13321569561958313,-0.00024480317370034754,-0.0073145609349012375,0.0003329376922920346,-0.000794418912846595,-0.007937678135931492,0.20841389894485474,-0.0001913720479933545,-0.0002066764427581802],[-0.0020480367820709944,-0.0003764008288271725,-0.0033146096393465996,-0.009665866382420063,-0.00031719700200483203,-0.005141303408890963,0.3171706795692444,0.002842522691935301,0.00047299303696490824,-0.0011535687372088432,0.27267059683799744,-0.00317572895437479],[-0.00043966571683995426,5.7257228036178276e-05,-0.002063408261165023,0.020066477358341217,-0.007870486006140709,0.01131565310060978,0.003057321300730109,0.06856381893157959,-0.0027472060173749924,-0.009278557263314724,0.0005067567108199,-0.0013152381870895624],[-0.012957250699400902,-0.00304490327835083,-0.017922621220350266,-0.004358463454991579,-0.0011523604625836015,0.0004998764488846064,-0.0031132360454648733,0.01958620175719261,4.3899446609430015e-05,0.012970171868801117,-0.007695660460740328,-0.0009143368806689978],[0.004100503865629435,-0.020459022372961044,-0.035875849425792694,0.014655501581728458,0.0008445944986306131,0.0017805789830163121,-0.018042270094156265,0.003519205143675208,0.008252425119280815,-0.0017664493061602116,0.044166937470436096,0.006474765483289957]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Logit Difference From Patched Head Value\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "<div>                            <div id=\"c769824a-8892-4cdc-ba58-adc59353a905\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"c769824a-8892-4cdc-ba58-adc59353a905\")) {                    Plotly.newPlot(                        \"c769824a-8892-4cdc-ba58-adc59353a905\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[-0.00019808451179414988,0.005339857656508684,0.0006520813331007957,0.0035054110921919346,-0.008983243256807327,0.003481481224298477,-0.0008612077799625695,-3.356226807227358e-05,0.0005169931682758033,0.000442283577285707,-0.003907017409801483,-0.0001884857047116384],[-0.0004399341996759176,-0.00044604254071600735,-6.719166412949562e-05,7.296437252080068e-05,-3.587806349969469e-05,-0.001932314014993608,-0.0001571049797348678,1.550576780573465e-05,0.00020597163529600948,0.0003376699751242995,0.00035139694227837026,-0.0005663632764481008],[0.0002103011793224141,-0.0007198435487225652,0.00048675358993932605,-0.0005971734644845128,-0.0005919041577726603,-0.0005438430234789848,-0.0002275186125189066,-0.00047960481606423855,0.00020617301925085485,0.0011842446401715279,-0.00035743816988542676,-0.0009093025582842529],[0.0010397254955023527,-0.0001205221051350236,-7.69247199059464e-05,-0.0007269251509569585,-0.001310002407990396,-0.002310896525159478,0.010987145826220512,-5.06454634887632e-05,0.00014364650996867567,0.00015129870735108852,-7.920695497887209e-05,-1.9667490050778724e-05],[-0.0005367277772165835,-0.0008130795322358608,-0.000133275767439045,0.030610064044594765,-0.007185581140220165,0.00014941922563593835,0.001333831693045795,-0.011422313749790192,-0.0005329688428901136,0.0005127307958900928,0.00037368229823186994,0.002956030424684286],[7.4508234320092015e-06,6.712453796353657e-06,0.0015982352197170258,0.00033918028930202127,-0.0012586521916091442,-5.423662514658645e-05,0.0006328836898319423,-0.0002699077595025301,7.430685946019366e-05,-0.006704063154757023,0.0031767694745212793,-0.001728456816636026],[0.04863159358501434,0.015315738506615162,-0.0004640990518964827,-0.0001160583269665949,-4.839679240831174e-05,-0.003953501116484404,-0.017375119030475616,-0.0001535809424240142,0.0012203240767121315,-0.00017975950322579592,-0.0004269791825208813,0.0001233748917002231],[-2.849436532414984e-05,-0.001384309260174632,-0.00012176390737295151,0.13321569561958313,-0.00024480317370034754,-0.0073145609349012375,0.0003329376922920346,-0.000794418912846595,-0.007937678135931492,0.20841389894485474,-0.0001913720479933545,-0.0002066764427581802],[-0.0020480367820709944,-0.0003764008288271725,-0.0033146096393465996,-0.009665866382420063,-0.00031719700200483203,-0.005141303408890963,0.3171706795692444,0.002842522691935301,0.00047299303696490824,-0.0011535687372088432,0.27267059683799744,-0.00317572895437479],[-0.00043966571683995426,5.7257228036178276e-05,-0.002063408261165023,0.020066477358341217,-0.007870486006140709,0.01131565310060978,0.003057321300730109,0.06856381893157959,-0.0027472060173749924,-0.009278557263314724,0.0005067567108199,-0.0013152381870895624],[-0.012957250699400902,-0.00304490327835083,-0.017922621220350266,-0.004358463454991579,-0.0011523604625836015,0.0004998764488846064,-0.0031132360454648733,0.01958620175719261,4.3899446609430015e-05,0.012970171868801117,-0.007695660460740328,-0.0009143368806689978],[0.004100503865629435,-0.020459022372961044,-0.035875849425792694,0.014655501581728458,0.0008445944986306131,0.0017805789830163121,-0.018042270094156265,0.003519205143675208,0.008252425119280815,-0.0017664493061602116,0.044166937470436096,0.006474765483289957]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Logit Difference From Patched Head Value\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('d15c92bc-9a5e-459f-a673-8ffed2367454');\n",
+       "var gd = document.getElementById('c769824a-8892-4cdc-ba58-adc59353a905');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -10600,10 +10600,10 @@
    "execution_count": 30,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:40:59.827603Z",
-     "iopub.status.busy": "2024-09-05T17:40:59.827260Z",
-     "iopub.status.idle": "2024-09-05T17:40:59.871179Z",
-     "shell.execute_reply": "2024-09-05T17:40:59.870598Z"
+     "iopub.execute_input": "2024-09-10T17:14:38.941169Z",
+     "iopub.status.busy": "2024-09-10T17:14:38.940830Z",
+     "iopub.status.idle": "2024-09-10T17:14:38.984539Z",
+     "shell.execute_reply": "2024-09-10T17:14:38.983972Z"
     }
    },
    "outputs": [
@@ -12117,9 +12117,9 @@
        }
       },
       "text/html": [
-       "<div>                            <div id=\"01dae98a-ebc8-4e3b-adb0-ff57e3b90a6d\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"01dae98a-ebc8-4e3b-adb0-ff57e3b90a6d\")) {                    Plotly.newPlot(                        \"01dae98a-ebc8-4e3b-adb0-ff57e3b90a6d\",                        [{\"hovertemplate\":\"\\u003cb\\u003e%{hovertext}\\u003c\\u002fb\\u003e\\u003cbr\\u003e\\u003cbr\\u003eValue Patch=%{x}\\u003cbr\\u003eOutput Patch=%{y}\\u003cbr\\u003eLayer=%{marker.color}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"hovertext\":[\"L0H0\",\"L0H1\",\"L0H2\",\"L0H3\",\"L0H4\",\"L0H5\",\"L0H6\",\"L0H7\",\"L0H8\",\"L0H9\",\"L0H10\",\"L0H11\",\"L1H0\",\"L1H1\",\"L1H2\",\"L1H3\",\"L1H4\",\"L1H5\",\"L1H6\",\"L1H7\",\"L1H8\",\"L1H9\",\"L1H10\",\"L1H11\",\"L2H0\",\"L2H1\",\"L2H2\",\"L2H3\",\"L2H4\",\"L2H5\",\"L2H6\",\"L2H7\",\"L2H8\",\"L2H9\",\"L2H10\",\"L2H11\",\"L3H0\",\"L3H1\",\"L3H2\",\"L3H3\",\"L3H4\",\"L3H5\",\"L3H6\",\"L3H7\",\"L3H8\",\"L3H9\",\"L3H10\",\"L3H11\",\"L4H0\",\"L4H1\",\"L4H2\",\"L4H3\",\"L4H4\",\"L4H5\",\"L4H6\",\"L4H7\",\"L4H8\",\"L4H9\",\"L4H10\",\"L4H11\",\"L5H0\",\"L5H1\",\"L5H2\",\"L5H3\",\"L5H4\",\"L5H5\",\"L5H6\",\"L5H7\",\"L5H8\",\"L5H9\",\"L5H10\",\"L5H11\",\"L6H0\",\"L6H1\",\"L6H2\",\"L6H3\",\"L6H4\",\"L6H5\",\"L6H6\",\"L6H7\",\"L6H8\",\"L6H9\",\"L6H10\",\"L6H11\",\"L7H0\",\"L7H1\",\"L7H2\",\"L7H3\",\"L7H4\",\"L7H5\",\"L7H6\",\"L7H7\",\"L7H8\",\"L7H9\",\"L7H10\",\"L7H11\",\"L8H0\",\"L8H1\",\"L8H2\",\"L8H3\",\"L8H4\",\"L8H5\",\"L8H6\",\"L8H7\",\"L8H8\",\"L8H9\",\"L8H10\",\"L8H11\",\"L9H0\",\"L9H1\",\"L9H2\",\"L9H3\",\"L9H4\",\"L9H5\",\"L9H6\",\"L9H7\",\"L9H8\",\"L9H9\",\"L9H10\",\"L9H11\",\"L10H0\",\"L10H1\",\"L10H2\",\"L10H3\",\"L10H4\",\"L10H5\",\"L10H6\",\"L10H7\",\"L10H8\",\"L10H9\",\"L10H10\",\"L10H11\",\"L11H0\",\"L11H1\",\"L11H2\",\"L11H3\",\"L11H4\",\"L11H5\",\"L11H6\",\"L11H7\",\"L11H8\",\"L11H9\",\"L11H10\",\"L11H11\"],\"legendgroup\":\"\",\"marker\":{\"color\":[0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4,4,4,4,5,5,5,5,5,5,5,5,5,5,5,5,6,6,6,6,6,6,6,6,6,6,6,6,7,7,7,7,7,7,7,7,7,7,7,7,8,8,8,8,8,8,8,8,8,8,8,8,9,9,9,9,9,9,9,9,9,9,9,9,10,10,10,10,10,10,10,10,10,10,10,10,11,11,11,11,11,11,11,11,11,11,11,11],\"coloraxis\":\"coloraxis\",\"symbol\":\"circle\"},\"mode\":\"markers\",\"name\":\"\",\"orientation\":\"v\",\"showlegend\":false,\"x\":[-0.00019808451179414988,0.005339857656508684,0.0006520813331007957,0.0035054110921919346,-0.008983243256807327,0.003481481224298477,-0.0008612077799625695,-3.356226807227358e-05,0.0005169931682758033,0.000442283577285707,-0.003907017409801483,-0.0001884857047116384,-0.0004399341996759176,-0.00044604254071600735,-6.719166412949562e-05,7.296437252080068e-05,-3.587806349969469e-05,-0.001932314014993608,-0.0001571049797348678,1.550576780573465e-05,0.00020597163529600948,0.0003376699751242995,0.00035139694227837026,-0.0005663632764481008,0.0002103011793224141,-0.0007198435487225652,0.00048675358993932605,-0.0005971734644845128,-0.0005919041577726603,-0.0005438430234789848,-0.0002275186125189066,-0.00047960481606423855,0.00020617301925085485,0.0011842446401715279,-0.00035743816988542676,-0.0009093025582842529,0.0010397254955023527,-0.0001205221051350236,-7.69247199059464e-05,-0.0007269251509569585,-0.001310002407990396,-0.002310896525159478,0.010987145826220512,-5.06454634887632e-05,0.00014364650996867567,0.00015129870735108852,-7.920695497887209e-05,-1.9667490050778724e-05,-0.0005367277772165835,-0.0008130795322358608,-0.000133275767439045,0.030610064044594765,-0.007185581140220165,0.00014941922563593835,0.001333831693045795,-0.011422313749790192,-0.0005329688428901136,0.0005127307958900928,0.00037368229823186994,0.002956030424684286,7.4508234320092015e-06,6.712453796353657e-06,0.0015982352197170258,0.00033918028930202127,-0.0012586521916091442,-5.423662514658645e-05,0.0006328836898319423,-0.0002699077595025301,7.430685946019366e-05,-0.006704063154757023,0.0031767694745212793,-0.001728456816636026,0.04863159358501434,0.015315738506615162,-0.0004640990518964827,-0.0001160583269665949,-4.839679240831174e-05,-0.003953501116484404,-0.017375119030475616,-0.0001535809424240142,0.0012203240767121315,-0.00017975950322579592,-0.0004269791825208813,0.0001233748917002231,-2.849436532414984e-05,-0.001384309260174632,-0.00012176390737295151,0.13321569561958313,-0.00024480317370034754,-0.0073145609349012375,0.0003329376922920346,-0.000794418912846595,-0.007937678135931492,0.20841389894485474,-0.0001913720479933545,-0.0002066764427581802,-0.0020480367820709944,-0.0003764008288271725,-0.0033146096393465996,-0.009665866382420063,-0.00031719700200483203,-0.005141303408890963,0.3171706795692444,0.002842522691935301,0.00047299303696490824,-0.0011535687372088432,0.27267059683799744,-0.00317572895437479,-0.00043966571683995426,5.7257228036178276e-05,-0.002063408261165023,0.020066477358341217,-0.007870486006140709,0.01131565310060978,0.003057321300730109,0.06856381893157959,-0.0027472060173749924,-0.009278557263314724,0.0005067567108199,-0.0013152381870895624,-0.012957250699400902,-0.00304490327835083,-0.017922621220350266,-0.004358463454991579,-0.0011523604625836015,0.0004998764488846064,-0.0031132360454648733,0.01958620175719261,4.3899446609430015e-05,0.012970171868801117,-0.007695660460740328,-0.0009143368806689978,0.004100503865629435,-0.020459022372961044,-0.035875849425792694,0.014655501581728458,0.0008445944986306131,0.0017805789830163121,-0.018042270094156265,0.003519205143675208,0.008252425119280815,-0.0017664493061602116,0.044166937470436096,0.006474765483289957],\"xaxis\":\"x\",\"y\":[0.000949342327658087,0.016124119982123375,0.0018556242575868964,0.0034389242064207792,-0.009823138825595379,0.011059673503041267,-0.0040638200007379055,-0.0015785341383889318,-0.001208073808811605,0.003828749991953373,-0.004257441032677889,-0.0011428623693063855,-0.001076644053682685,-0.0003779782564379275,3.0206042538338806e-06,-0.000260510336374864,-0.00014149851631373167,0.0038328110240399837,-0.0004281202855054289,-0.0014291484840214252,-0.0009217541082762182,0.0006938663427717984,0.00043359093251638114,-0.0035708239302039146,-0.0004966544220224023,0.0008070718613453209,0.0005420306115411222,-0.0005315592279657722,-0.0007152119651436806,-0.0010380810126662254,-0.0009492752142250538,-8.538240945199504e-05,0.0002774928288999945,0.0021076432894915342,-0.0001976146304514259,-0.0016398524167016149,0.11626167595386505,0.00025017315056174994,-0.0014669395750388503,-0.0003969745012000203,0.01896214485168457,-0.00018882131553255022,0.011170395649969578,-0.0013304082676768303,-0.0007363561890088022,-0.00030310085276141763,-0.00014579448907170445,-0.0002225178322987631,-0.0016502231592312455,0.00029299859306775033,-0.0014352232683449984,0.030841777101159096,-0.007431894540786743,-0.0002820908557623625,0.006017110776156187,-0.011006947606801987,-0.0012651296565309167,0.001490030437707901,-0.00018016225658357143,0.0029450219590216875,-0.004210856277495623,0.0029608632903546095,0.002045620232820511,0.0013410139363259077,-0.0012189144035801291,0.34349939227104187,0.0005627049831673503,-0.0001257913827430457,-0.005152345169335604,0.016241183504462242,0.017089638859033585,-0.004173669498413801,0.039775215089321136,0.015227536670863628,-0.001023011514917016,0.0008081123232841492,-0.004935599863529205,-0.002123551908880472,-0.014273260720074177,0.0013749118661507964,0.0014838549541309476,0.1302708089351654,-0.00033538774005137384,0.0012914760736748576,0.0003724069392774254,0.0195147804915905,0.00022308839834295213,0.12424899637699127,-0.0004041232750751078,-0.007652163505554199,0.0013012762647122145,-0.0011244030902162194,-0.007449581753462553,0.19224178791046143,-0.0032753082923591137,-0.000501957256346941,-0.001007740618661046,3.1212908652378246e-05,-0.0008586570620536804,0.012360110878944397,-0.00040429108776152134,-0.004328995477408171,0.31855517625808716,0.002329523442313075,0.002118081087246537,0.0001410957775078714,0.27793484926223755,0.005738543812185526,0.0058897752314805984,-0.000968372100032866,0.009125681594014168,0.020675498992204666,-0.03700448200106621,0.014263628050684929,-0.04828385263681412,0.05834108963608742,0.0006516450084745884,0.2636081576347351,0.0004918886115774512,-0.0026110773906111717,0.08374276012182236,0.020676370710134506,-0.0037437702994793653,0.01085111778229475,-0.0010960765648633242,0.0004738992138300091,0.04817944020032883,-0.47990989685058594,0.00018412260396871716,0.011861777864396572,0.06088537722826004,0.0008459034143015742,0.005328110884875059,-0.011493667028844357,-0.11350765824317932,0.006330112461000681,0.0003166264505125582,-0.0011600126745179296,-0.022669030353426933,0.004070901777595282,0.007316238712519407,-0.008345123380422592,-0.27817434072494507,0.003635196480900049],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Value Patch\"},\"range\":[-0.5,0.5]},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Output Patch\"},\"range\":[-0.5,0.5]},\"coloraxis\":{\"colorbar\":{\"title\":{\"text\":\"Layer\"}},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]},\"legend\":{\"tracegroupgap\":0},\"title\":{\"text\":\"Scatter plot of output patching vs value patching\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "<div>                            <div id=\"1c9555de-e7b8-41d2-973b-2deffd53bf8d\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"1c9555de-e7b8-41d2-973b-2deffd53bf8d\")) {                    Plotly.newPlot(                        \"1c9555de-e7b8-41d2-973b-2deffd53bf8d\",                        [{\"hovertemplate\":\"\\u003cb\\u003e%{hovertext}\\u003c\\u002fb\\u003e\\u003cbr\\u003e\\u003cbr\\u003eValue Patch=%{x}\\u003cbr\\u003eOutput Patch=%{y}\\u003cbr\\u003eLayer=%{marker.color}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"hovertext\":[\"L0H0\",\"L0H1\",\"L0H2\",\"L0H3\",\"L0H4\",\"L0H5\",\"L0H6\",\"L0H7\",\"L0H8\",\"L0H9\",\"L0H10\",\"L0H11\",\"L1H0\",\"L1H1\",\"L1H2\",\"L1H3\",\"L1H4\",\"L1H5\",\"L1H6\",\"L1H7\",\"L1H8\",\"L1H9\",\"L1H10\",\"L1H11\",\"L2H0\",\"L2H1\",\"L2H2\",\"L2H3\",\"L2H4\",\"L2H5\",\"L2H6\",\"L2H7\",\"L2H8\",\"L2H9\",\"L2H10\",\"L2H11\",\"L3H0\",\"L3H1\",\"L3H2\",\"L3H3\",\"L3H4\",\"L3H5\",\"L3H6\",\"L3H7\",\"L3H8\",\"L3H9\",\"L3H10\",\"L3H11\",\"L4H0\",\"L4H1\",\"L4H2\",\"L4H3\",\"L4H4\",\"L4H5\",\"L4H6\",\"L4H7\",\"L4H8\",\"L4H9\",\"L4H10\",\"L4H11\",\"L5H0\",\"L5H1\",\"L5H2\",\"L5H3\",\"L5H4\",\"L5H5\",\"L5H6\",\"L5H7\",\"L5H8\",\"L5H9\",\"L5H10\",\"L5H11\",\"L6H0\",\"L6H1\",\"L6H2\",\"L6H3\",\"L6H4\",\"L6H5\",\"L6H6\",\"L6H7\",\"L6H8\",\"L6H9\",\"L6H10\",\"L6H11\",\"L7H0\",\"L7H1\",\"L7H2\",\"L7H3\",\"L7H4\",\"L7H5\",\"L7H6\",\"L7H7\",\"L7H8\",\"L7H9\",\"L7H10\",\"L7H11\",\"L8H0\",\"L8H1\",\"L8H2\",\"L8H3\",\"L8H4\",\"L8H5\",\"L8H6\",\"L8H7\",\"L8H8\",\"L8H9\",\"L8H10\",\"L8H11\",\"L9H0\",\"L9H1\",\"L9H2\",\"L9H3\",\"L9H4\",\"L9H5\",\"L9H6\",\"L9H7\",\"L9H8\",\"L9H9\",\"L9H10\",\"L9H11\",\"L10H0\",\"L10H1\",\"L10H2\",\"L10H3\",\"L10H4\",\"L10H5\",\"L10H6\",\"L10H7\",\"L10H8\",\"L10H9\",\"L10H10\",\"L10H11\",\"L11H0\",\"L11H1\",\"L11H2\",\"L11H3\",\"L11H4\",\"L11H5\",\"L11H6\",\"L11H7\",\"L11H8\",\"L11H9\",\"L11H10\",\"L11H11\"],\"legendgroup\":\"\",\"marker\":{\"color\":[0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4,4,4,4,5,5,5,5,5,5,5,5,5,5,5,5,6,6,6,6,6,6,6,6,6,6,6,6,7,7,7,7,7,7,7,7,7,7,7,7,8,8,8,8,8,8,8,8,8,8,8,8,9,9,9,9,9,9,9,9,9,9,9,9,10,10,10,10,10,10,10,10,10,10,10,10,11,11,11,11,11,11,11,11,11,11,11,11],\"coloraxis\":\"coloraxis\",\"symbol\":\"circle\"},\"mode\":\"markers\",\"name\":\"\",\"orientation\":\"v\",\"showlegend\":false,\"x\":[-0.00019808451179414988,0.005339857656508684,0.0006520813331007957,0.0035054110921919346,-0.008983243256807327,0.003481481224298477,-0.0008612077799625695,-3.356226807227358e-05,0.0005169931682758033,0.000442283577285707,-0.003907017409801483,-0.0001884857047116384,-0.0004399341996759176,-0.00044604254071600735,-6.719166412949562e-05,7.296437252080068e-05,-3.587806349969469e-05,-0.001932314014993608,-0.0001571049797348678,1.550576780573465e-05,0.00020597163529600948,0.0003376699751242995,0.00035139694227837026,-0.0005663632764481008,0.0002103011793224141,-0.0007198435487225652,0.00048675358993932605,-0.0005971734644845128,-0.0005919041577726603,-0.0005438430234789848,-0.0002275186125189066,-0.00047960481606423855,0.00020617301925085485,0.0011842446401715279,-0.00035743816988542676,-0.0009093025582842529,0.0010397254955023527,-0.0001205221051350236,-7.69247199059464e-05,-0.0007269251509569585,-0.001310002407990396,-0.002310896525159478,0.010987145826220512,-5.06454634887632e-05,0.00014364650996867567,0.00015129870735108852,-7.920695497887209e-05,-1.9667490050778724e-05,-0.0005367277772165835,-0.0008130795322358608,-0.000133275767439045,0.030610064044594765,-0.007185581140220165,0.00014941922563593835,0.001333831693045795,-0.011422313749790192,-0.0005329688428901136,0.0005127307958900928,0.00037368229823186994,0.002956030424684286,7.4508234320092015e-06,6.712453796353657e-06,0.0015982352197170258,0.00033918028930202127,-0.0012586521916091442,-5.423662514658645e-05,0.0006328836898319423,-0.0002699077595025301,7.430685946019366e-05,-0.006704063154757023,0.0031767694745212793,-0.001728456816636026,0.04863159358501434,0.015315738506615162,-0.0004640990518964827,-0.0001160583269665949,-4.839679240831174e-05,-0.003953501116484404,-0.017375119030475616,-0.0001535809424240142,0.0012203240767121315,-0.00017975950322579592,-0.0004269791825208813,0.0001233748917002231,-2.849436532414984e-05,-0.001384309260174632,-0.00012176390737295151,0.13321569561958313,-0.00024480317370034754,-0.0073145609349012375,0.0003329376922920346,-0.000794418912846595,-0.007937678135931492,0.20841389894485474,-0.0001913720479933545,-0.0002066764427581802,-0.0020480367820709944,-0.0003764008288271725,-0.0033146096393465996,-0.009665866382420063,-0.00031719700200483203,-0.005141303408890963,0.3171706795692444,0.002842522691935301,0.00047299303696490824,-0.0011535687372088432,0.27267059683799744,-0.00317572895437479,-0.00043966571683995426,5.7257228036178276e-05,-0.002063408261165023,0.020066477358341217,-0.007870486006140709,0.01131565310060978,0.003057321300730109,0.06856381893157959,-0.0027472060173749924,-0.009278557263314724,0.0005067567108199,-0.0013152381870895624,-0.012957250699400902,-0.00304490327835083,-0.017922621220350266,-0.004358463454991579,-0.0011523604625836015,0.0004998764488846064,-0.0031132360454648733,0.01958620175719261,4.3899446609430015e-05,0.012970171868801117,-0.007695660460740328,-0.0009143368806689978,0.004100503865629435,-0.020459022372961044,-0.035875849425792694,0.014655501581728458,0.0008445944986306131,0.0017805789830163121,-0.018042270094156265,0.003519205143675208,0.008252425119280815,-0.0017664493061602116,0.044166937470436096,0.006474765483289957],\"xaxis\":\"x\",\"y\":[0.000949342327658087,0.016124119982123375,0.0018556242575868964,0.0034389242064207792,-0.009823138825595379,0.011059673503041267,-0.0040638200007379055,-0.0015785341383889318,-0.001208073808811605,0.003828749991953373,-0.004257441032677889,-0.0011428623693063855,-0.001076644053682685,-0.0003779782564379275,3.0206042538338806e-06,-0.000260510336374864,-0.00014149851631373167,0.0038328110240399837,-0.0004281202855054289,-0.0014291484840214252,-0.0009217541082762182,0.0006938663427717984,0.00043359093251638114,-0.0035708239302039146,-0.0004966544220224023,0.0008070718613453209,0.0005420306115411222,-0.0005315592279657722,-0.0007152119651436806,-0.0010380810126662254,-0.0009492752142250538,-8.538240945199504e-05,0.0002774928288999945,0.0021076432894915342,-0.0001976146304514259,-0.0016398524167016149,0.11626167595386505,0.00025017315056174994,-0.0014669395750388503,-0.0003969745012000203,0.01896214485168457,-0.00018882131553255022,0.011170395649969578,-0.0013304082676768303,-0.0007363561890088022,-0.00030310085276141763,-0.00014579448907170445,-0.0002225178322987631,-0.0016502231592312455,0.00029299859306775033,-0.0014352232683449984,0.030841777101159096,-0.007431894540786743,-0.0002820908557623625,0.006017110776156187,-0.011006947606801987,-0.0012651296565309167,0.001490030437707901,-0.00018016225658357143,0.0029450219590216875,-0.004210856277495623,0.0029608632903546095,0.002045620232820511,0.0013410139363259077,-0.0012189144035801291,0.34349939227104187,0.0005627049831673503,-0.0001257913827430457,-0.005152345169335604,0.016241183504462242,0.017089638859033585,-0.004173669498413801,0.039775215089321136,0.015227536670863628,-0.001023011514917016,0.0008081123232841492,-0.004935599863529205,-0.002123551908880472,-0.014273260720074177,0.0013749118661507964,0.0014838549541309476,0.1302708089351654,-0.00033538774005137384,0.0012914760736748576,0.0003724069392774254,0.0195147804915905,0.00022308839834295213,0.12424899637699127,-0.0004041232750751078,-0.007652163505554199,0.0013012762647122145,-0.0011244030902162194,-0.007449581753462553,0.19224178791046143,-0.0032753082923591137,-0.000501957256346941,-0.001007740618661046,3.1212908652378246e-05,-0.0008586570620536804,0.012360110878944397,-0.00040429108776152134,-0.004328995477408171,0.31855517625808716,0.002329523442313075,0.002118081087246537,0.0001410957775078714,0.27793484926223755,0.005738543812185526,0.0058897752314805984,-0.000968372100032866,0.009125681594014168,0.020675498992204666,-0.03700448200106621,0.014263628050684929,-0.04828385263681412,0.05834108963608742,0.0006516450084745884,0.2636081576347351,0.0004918886115774512,-0.0026110773906111717,0.08374276012182236,0.020676370710134506,-0.0037437702994793653,0.01085111778229475,-0.0010960765648633242,0.0004738992138300091,0.04817944020032883,-0.47990989685058594,0.00018412260396871716,0.011861777864396572,0.06088537722826004,0.0008459034143015742,0.005328110884875059,-0.011493667028844357,-0.11350765824317932,0.006330112461000681,0.0003166264505125582,-0.0011600126745179296,-0.022669030353426933,0.004070901777595282,0.007316238712519407,-0.008345123380422592,-0.27817434072494507,0.003635196480900049],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Value Patch\"},\"range\":[-0.5,0.5]},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Output Patch\"},\"range\":[-0.5,0.5]},\"coloraxis\":{\"colorbar\":{\"title\":{\"text\":\"Layer\"}},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]},\"legend\":{\"tracegroupgap\":0},\"title\":{\"text\":\"Scatter plot of output patching vs value patching\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('01dae98a-ebc8-4e3b-adb0-ff57e3b90a6d');\n",
+       "var gd = document.getElementById('1c9555de-e7b8-41d2-973b-2deffd53bf8d');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -12181,10 +12181,10 @@
    "execution_count": 31,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:40:59.873511Z",
-     "iopub.status.busy": "2024-09-05T17:40:59.873067Z",
-     "iopub.status.idle": "2024-09-05T17:41:38.016202Z",
-     "shell.execute_reply": "2024-09-05T17:41:38.015636Z"
+     "iopub.execute_input": "2024-09-10T17:14:38.986761Z",
+     "iopub.status.busy": "2024-09-10T17:14:38.986446Z",
+     "iopub.status.idle": "2024-09-10T17:15:16.985652Z",
+     "shell.execute_reply": "2024-09-10T17:15:16.985000Z"
     }
    },
    "outputs": [],
@@ -12224,10 +12224,10 @@
    "execution_count": 32,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:41:38.018845Z",
-     "iopub.status.busy": "2024-09-05T17:41:38.018454Z",
-     "iopub.status.idle": "2024-09-05T17:41:38.077603Z",
-     "shell.execute_reply": "2024-09-05T17:41:38.076989Z"
+     "iopub.execute_input": "2024-09-10T17:15:16.988321Z",
+     "iopub.status.busy": "2024-09-10T17:15:16.987969Z",
+     "iopub.status.idle": "2024-09-10T17:15:17.046487Z",
+     "shell.execute_reply": "2024-09-10T17:15:17.045970Z"
     }
    },
    "outputs": [
@@ -13313,9 +13313,9 @@
        }
       },
       "text/html": [
-       "<div>                            <div id=\"19f79480-4a62-4c7c-977d-3dc9f4e33aff\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"19f79480-4a62-4c7c-977d-3dc9f4e33aff\")) {                    Plotly.newPlot(                        \"19f79480-4a62-4c7c-977d-3dc9f4e33aff\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.0006398981786333025,0.005319283809512854,0.0011588380439206958,-5.923740172875114e-05,-0.0010669109178707004,0.005080018658190966,-0.0030812176410108805,-0.0020524668507277966,-0.0014406939735636115,0.003492556745186448,-0.0025678491219878197,-0.0009152094717137516],[-0.0007601853576488793,0.0001696908293524757,0.00012250228610355407,-0.00034918184974230826,1.5405081285280176e-05,0.005008597858250141,-0.0002970260684378445,-0.0014443857362493873,-0.001099835499189794,0.00047436909517273307,5.108177356305532e-05,-0.003494771895930171],[-0.0007234682561829686,0.0017475537024438381,-0.0001551919267512858,5.749216506956145e-05,-9.699495421955362e-05,-0.00042355581535957754,-0.000791196885984391,0.0002723913639783859,0.00010212998313363642,0.00042187771759927273,0.00015183570212684572,-0.0007432028651237488],[0.11458060890436172,0.00021201284835115075,-0.000942025741096586,0.0004294627869967371,0.02004331909120083,0.002104991814121604,7.662265852559358e-05,-0.0015435622772201896,-0.0008476486545987427,-0.0005817683413624763,0.00011934742360608652,-1.9130493456032127e-05],[-0.001126819639466703,0.0012377764796838164,-0.0012320708483457565,-0.0005953610525466502,-0.0007538085337728262,-0.0005843526450917125,0.004813097883015871,0.00018163899949286133,-0.0005363250384107232,0.0008580865105614066,-0.00029840212664566934,-1.2183103535789996e-05],[-0.004240894690155983,0.0029526741709560156,0.0005220275488682091,0.0009532691328786314,0.0001625084987608716,0.3435087502002716,-0.0003038727736566216,0.00010364028275944293,-0.005300321150571108,0.024866117164492607,0.014383613131940365,-0.0023283488117158413],[-0.0023879888467490673,-0.002172317821532488,-0.00047651707427576184,0.0004340272571425885,-0.004673780873417854,0.001858678413555026,-0.0026543056592345238,0.0014364985981956124,0.000303067296044901,0.1304299533367157,8.957769750850275e-05,0.0011777335312217474],[0.0003194456803612411,0.020570717751979828,0.00031887509976513684,-0.002512404229491949,-0.0002621884341351688,-0.00024587716325186193,0.0005522335413843393,-0.00043134228326380253,0.0002570534124970436,0.008090788498520851,-0.003069269470870495,-0.0004241599526721984],[0.0009768297895789146,0.00039294702582992613,0.0017541655106469989,0.022596266120672226,-4.443644138518721e-05,0.00014159921556711197,0.00958488043397665,-0.0003154182049911469,0.0015266804257407784,0.0011816268088296056,-0.01077402476221323,0.009365785866975784],[0.006314304657280445,-0.001094431965611875,0.011661848053336143,0.0013478941982612014,-0.02918611839413643,0.0038340529426932335,-0.04409320279955864,-0.005031823180615902,0.004822427872568369,0.2766473889350891,-3.1280032999347895e-05,-0.0006619150517508388],[0.09538879990577698,0.025069067254662514,0.014238725416362286,0.01475451048463583,9.914294059854001e-05,-8.977906691143289e-05,0.0508294515311718,-0.5050979852676392,0.00014717054727952927,-0.0016020277980715036,0.06883121281862259,0.0023271406535059214],[0.0013427927624434233,0.009631095454096794,-0.07776330411434174,-0.007728920318186283,-0.0005721024353988469,-0.0029568022582679987,-0.00494667561724782,0.000458796217571944,-0.0006325145368464291,-0.006519974209368229,-0.32048866152763367,-0.002472163178026676]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Logit Difference From Patched Head Pattern\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "<div>                            <div id=\"ffed76f4-77df-4501-b30f-4cced68f4dbf\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"ffed76f4-77df-4501-b30f-4cced68f4dbf\")) {                    Plotly.newPlot(                        \"ffed76f4-77df-4501-b30f-4cced68f4dbf\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.0006398981786333025,0.005319283809512854,0.0011588380439206958,-5.923740172875114e-05,-0.0010669109178707004,0.005080018658190966,-0.0030812176410108805,-0.0020524668507277966,-0.0014406939735636115,0.003492556745186448,-0.0025678491219878197,-0.0009152094717137516],[-0.0007601853576488793,0.0001696908293524757,0.00012250228610355407,-0.00034918184974230826,1.5405081285280176e-05,0.005008597858250141,-0.0002970260684378445,-0.0014443857362493873,-0.001099835499189794,0.00047436909517273307,5.108177356305532e-05,-0.003494771895930171],[-0.0007234682561829686,0.0017475537024438381,-0.0001551919267512858,5.749216506956145e-05,-9.699495421955362e-05,-0.00042355581535957754,-0.000791196885984391,0.0002723913639783859,0.00010212998313363642,0.00042187771759927273,0.00015183570212684572,-0.0007432028651237488],[0.11458060890436172,0.00021201284835115075,-0.000942025741096586,0.0004294627869967371,0.02004331909120083,0.002104991814121604,7.662265852559358e-05,-0.0015435622772201896,-0.0008476486545987427,-0.0005817683413624763,0.00011934742360608652,-1.9130493456032127e-05],[-0.001126819639466703,0.0012377764796838164,-0.0012320708483457565,-0.0005953610525466502,-0.0007538085337728262,-0.0005843526450917125,0.004813097883015871,0.00018163899949286133,-0.0005363250384107232,0.0008580865105614066,-0.00029840212664566934,-1.2183103535789996e-05],[-0.004240894690155983,0.0029526741709560156,0.0005220275488682091,0.0009532691328786314,0.0001625084987608716,0.3435087502002716,-0.0003038727736566216,0.00010364028275944293,-0.005300321150571108,0.024866117164492607,0.014383613131940365,-0.0023283488117158413],[-0.0023879888467490673,-0.002172317821532488,-0.00047651707427576184,0.0004340272571425885,-0.004673780873417854,0.001858678413555026,-0.0026543056592345238,0.0014364985981956124,0.000303067296044901,0.1304299533367157,8.957769750850275e-05,0.0011777335312217474],[0.0003194456803612411,0.020570717751979828,0.00031887509976513684,-0.002512404229491949,-0.0002621884341351688,-0.00024587716325186193,0.0005522335413843393,-0.00043134228326380253,0.0002570534124970436,0.008090788498520851,-0.003069269470870495,-0.0004241599526721984],[0.0009768297895789146,0.00039294702582992613,0.0017541655106469989,0.022596266120672226,-4.443644138518721e-05,0.00014159921556711197,0.00958488043397665,-0.0003154182049911469,0.0015266804257407784,0.0011816268088296056,-0.01077402476221323,0.009365785866975784],[0.006314304657280445,-0.001094431965611875,0.011661848053336143,0.0013478941982612014,-0.02918611839413643,0.0038340529426932335,-0.04409320279955864,-0.005031823180615902,0.004822427872568369,0.2766473889350891,-3.1280032999347895e-05,-0.0006619150517508388],[0.09538879990577698,0.025069067254662514,0.014238725416362286,0.01475451048463583,9.914294059854001e-05,-8.977906691143289e-05,0.0508294515311718,-0.5050979852676392,0.00014717054727952927,-0.0016020277980715036,0.06883121281862259,0.0023271406535059214],[0.0013427927624434233,0.009631095454096794,-0.07776330411434174,-0.007728920318186283,-0.0005721024353988469,-0.0029568022582679987,-0.00494667561724782,0.000458796217571944,-0.0006325145368464291,-0.006519974209368229,-0.32048866152763367,-0.002472163178026676]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Logit Difference From Patched Head Pattern\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('19f79480-4a62-4c7c-977d-3dc9f4e33aff');\n",
+       "var gd = document.getElementById('ffed76f4-77df-4501-b30f-4cced68f4dbf');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -14650,9 +14650,9 @@
        }
       },
       "text/html": [
-       "<div>                            <div id=\"f93bc8e7-172e-489a-86d5-ed9f1696b43b\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"f93bc8e7-172e-489a-86d5-ed9f1696b43b\")) {                    Plotly.newPlot(                        \"f93bc8e7-172e-489a-86d5-ed9f1696b43b\",                        [{\"hovertemplate\":\"\\u003cb\\u003e%{hovertext}\\u003c\\u002fb\\u003e\\u003cbr\\u003e\\u003cbr\\u003eAttention Patch=%{x}\\u003cbr\\u003eOutput Patch=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"hovertext\":[\"L0H0\",\"L0H1\",\"L0H2\",\"L0H3\",\"L0H4\",\"L0H5\",\"L0H6\",\"L0H7\",\"L0H8\",\"L0H9\",\"L0H10\",\"L0H11\",\"L1H0\",\"L1H1\",\"L1H2\",\"L1H3\",\"L1H4\",\"L1H5\",\"L1H6\",\"L1H7\",\"L1H8\",\"L1H9\",\"L1H10\",\"L1H11\",\"L2H0\",\"L2H1\",\"L2H2\",\"L2H3\",\"L2H4\",\"L2H5\",\"L2H6\",\"L2H7\",\"L2H8\",\"L2H9\",\"L2H10\",\"L2H11\",\"L3H0\",\"L3H1\",\"L3H2\",\"L3H3\",\"L3H4\",\"L3H5\",\"L3H6\",\"L3H7\",\"L3H8\",\"L3H9\",\"L3H10\",\"L3H11\",\"L4H0\",\"L4H1\",\"L4H2\",\"L4H3\",\"L4H4\",\"L4H5\",\"L4H6\",\"L4H7\",\"L4H8\",\"L4H9\",\"L4H10\",\"L4H11\",\"L5H0\",\"L5H1\",\"L5H2\",\"L5H3\",\"L5H4\",\"L5H5\",\"L5H6\",\"L5H7\",\"L5H8\",\"L5H9\",\"L5H10\",\"L5H11\",\"L6H0\",\"L6H1\",\"L6H2\",\"L6H3\",\"L6H4\",\"L6H5\",\"L6H6\",\"L6H7\",\"L6H8\",\"L6H9\",\"L6H10\",\"L6H11\",\"L7H0\",\"L7H1\",\"L7H2\",\"L7H3\",\"L7H4\",\"L7H5\",\"L7H6\",\"L7H7\",\"L7H8\",\"L7H9\",\"L7H10\",\"L7H11\",\"L8H0\",\"L8H1\",\"L8H2\",\"L8H3\",\"L8H4\",\"L8H5\",\"L8H6\",\"L8H7\",\"L8H8\",\"L8H9\",\"L8H10\",\"L8H11\",\"L9H0\",\"L9H1\",\"L9H2\",\"L9H3\",\"L9H4\",\"L9H5\",\"L9H6\",\"L9H7\",\"L9H8\",\"L9H9\",\"L9H10\",\"L9H11\",\"L10H0\",\"L10H1\",\"L10H2\",\"L10H3\",\"L10H4\",\"L10H5\",\"L10H6\",\"L10H7\",\"L10H8\",\"L10H9\",\"L10H10\",\"L10H11\",\"L11H0\",\"L11H1\",\"L11H2\",\"L11H3\",\"L11H4\",\"L11H5\",\"L11H6\",\"L11H7\",\"L11H8\",\"L11H9\",\"L11H10\",\"L11H11\"],\"legendgroup\":\"\",\"marker\":{\"color\":\"#636efa\",\"symbol\":\"circle\"},\"mode\":\"markers\",\"name\":\"\",\"orientation\":\"v\",\"showlegend\":false,\"x\":[0.0006398981786333025,0.005319283809512854,0.0011588380439206958,-5.923740172875114e-05,-0.0010669109178707004,0.005080018658190966,-0.0030812176410108805,-0.0020524668507277966,-0.0014406939735636115,0.003492556745186448,-0.0025678491219878197,-0.0009152094717137516,-0.0007601853576488793,0.0001696908293524757,0.00012250228610355407,-0.00034918184974230826,1.5405081285280176e-05,0.005008597858250141,-0.0002970260684378445,-0.0014443857362493873,-0.001099835499189794,0.00047436909517273307,5.108177356305532e-05,-0.003494771895930171,-0.0007234682561829686,0.0017475537024438381,-0.0001551919267512858,5.749216506956145e-05,-9.699495421955362e-05,-0.00042355581535957754,-0.000791196885984391,0.0002723913639783859,0.00010212998313363642,0.00042187771759927273,0.00015183570212684572,-0.0007432028651237488,0.11458060890436172,0.00021201284835115075,-0.000942025741096586,0.0004294627869967371,0.02004331909120083,0.002104991814121604,7.662265852559358e-05,-0.0015435622772201896,-0.0008476486545987427,-0.0005817683413624763,0.00011934742360608652,-1.9130493456032127e-05,-0.001126819639466703,0.0012377764796838164,-0.0012320708483457565,-0.0005953610525466502,-0.0007538085337728262,-0.0005843526450917125,0.004813097883015871,0.00018163899949286133,-0.0005363250384107232,0.0008580865105614066,-0.00029840212664566934,-1.2183103535789996e-05,-0.004240894690155983,0.0029526741709560156,0.0005220275488682091,0.0009532691328786314,0.0001625084987608716,0.3435087502002716,-0.0003038727736566216,0.00010364028275944293,-0.005300321150571108,0.024866117164492607,0.014383613131940365,-0.0023283488117158413,-0.0023879888467490673,-0.002172317821532488,-0.00047651707427576184,0.0004340272571425885,-0.004673780873417854,0.001858678413555026,-0.0026543056592345238,0.0014364985981956124,0.000303067296044901,0.1304299533367157,8.957769750850275e-05,0.0011777335312217474,0.0003194456803612411,0.020570717751979828,0.00031887509976513684,-0.002512404229491949,-0.0002621884341351688,-0.00024587716325186193,0.0005522335413843393,-0.00043134228326380253,0.0002570534124970436,0.008090788498520851,-0.003069269470870495,-0.0004241599526721984,0.0009768297895789146,0.00039294702582992613,0.0017541655106469989,0.022596266120672226,-4.443644138518721e-05,0.00014159921556711197,0.00958488043397665,-0.0003154182049911469,0.0015266804257407784,0.0011816268088296056,-0.01077402476221323,0.009365785866975784,0.006314304657280445,-0.001094431965611875,0.011661848053336143,0.0013478941982612014,-0.02918611839413643,0.0038340529426932335,-0.04409320279955864,-0.005031823180615902,0.004822427872568369,0.2766473889350891,-3.1280032999347895e-05,-0.0006619150517508388,0.09538879990577698,0.025069067254662514,0.014238725416362286,0.01475451048463583,9.914294059854001e-05,-8.977906691143289e-05,0.0508294515311718,-0.5050979852676392,0.00014717054727952927,-0.0016020277980715036,0.06883121281862259,0.0023271406535059214,0.0013427927624434233,0.009631095454096794,-0.07776330411434174,-0.007728920318186283,-0.0005721024353988469,-0.0029568022582679987,-0.00494667561724782,0.000458796217571944,-0.0006325145368464291,-0.006519974209368229,-0.32048866152763367,-0.002472163178026676],\"xaxis\":\"x\",\"y\":[0.000949342327658087,0.016124119982123375,0.0018556242575868964,0.0034389242064207792,-0.009823138825595379,0.011059673503041267,-0.0040638200007379055,-0.0015785341383889318,-0.001208073808811605,0.003828749991953373,-0.004257441032677889,-0.0011428623693063855,-0.001076644053682685,-0.0003779782564379275,3.0206042538338806e-06,-0.000260510336374864,-0.00014149851631373167,0.0038328110240399837,-0.0004281202855054289,-0.0014291484840214252,-0.0009217541082762182,0.0006938663427717984,0.00043359093251638114,-0.0035708239302039146,-0.0004966544220224023,0.0008070718613453209,0.0005420306115411222,-0.0005315592279657722,-0.0007152119651436806,-0.0010380810126662254,-0.0009492752142250538,-8.538240945199504e-05,0.0002774928288999945,0.0021076432894915342,-0.0001976146304514259,-0.0016398524167016149,0.11626167595386505,0.00025017315056174994,-0.0014669395750388503,-0.0003969745012000203,0.01896214485168457,-0.00018882131553255022,0.011170395649969578,-0.0013304082676768303,-0.0007363561890088022,-0.00030310085276141763,-0.00014579448907170445,-0.0002225178322987631,-0.0016502231592312455,0.00029299859306775033,-0.0014352232683449984,0.030841777101159096,-0.007431894540786743,-0.0002820908557623625,0.006017110776156187,-0.011006947606801987,-0.0012651296565309167,0.001490030437707901,-0.00018016225658357143,0.0029450219590216875,-0.004210856277495623,0.0029608632903546095,0.002045620232820511,0.0013410139363259077,-0.0012189144035801291,0.34349939227104187,0.0005627049831673503,-0.0001257913827430457,-0.005152345169335604,0.016241183504462242,0.017089638859033585,-0.004173669498413801,0.039775215089321136,0.015227536670863628,-0.001023011514917016,0.0008081123232841492,-0.004935599863529205,-0.002123551908880472,-0.014273260720074177,0.0013749118661507964,0.0014838549541309476,0.1302708089351654,-0.00033538774005137384,0.0012914760736748576,0.0003724069392774254,0.0195147804915905,0.00022308839834295213,0.12424899637699127,-0.0004041232750751078,-0.007652163505554199,0.0013012762647122145,-0.0011244030902162194,-0.007449581753462553,0.19224178791046143,-0.0032753082923591137,-0.000501957256346941,-0.001007740618661046,3.1212908652378246e-05,-0.0008586570620536804,0.012360110878944397,-0.00040429108776152134,-0.004328995477408171,0.31855517625808716,0.002329523442313075,0.002118081087246537,0.0001410957775078714,0.27793484926223755,0.005738543812185526,0.0058897752314805984,-0.000968372100032866,0.009125681594014168,0.020675498992204666,-0.03700448200106621,0.014263628050684929,-0.04828385263681412,0.05834108963608742,0.0006516450084745884,0.2636081576347351,0.0004918886115774512,-0.0026110773906111717,0.08374276012182236,0.020676370710134506,-0.0037437702994793653,0.01085111778229475,-0.0010960765648633242,0.0004738992138300091,0.04817944020032883,-0.47990989685058594,0.00018412260396871716,0.011861777864396572,0.06088537722826004,0.0008459034143015742,0.005328110884875059,-0.011493667028844357,-0.11350765824317932,0.006330112461000681,0.0003166264505125582,-0.0011600126745179296,-0.022669030353426933,0.004070901777595282,0.007316238712519407,-0.008345123380422592,-0.27817434072494507,0.003635196480900049],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Attention Patch\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Output Patch\"}},\"legend\":{\"tracegroupgap\":0},\"title\":{\"text\":\"Scatter plot of output patching vs attention patching\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "<div>                            <div id=\"fc3a3636-e1a7-4d2b-b7d1-f71b5cc7bbb7\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"fc3a3636-e1a7-4d2b-b7d1-f71b5cc7bbb7\")) {                    Plotly.newPlot(                        \"fc3a3636-e1a7-4d2b-b7d1-f71b5cc7bbb7\",                        [{\"hovertemplate\":\"\\u003cb\\u003e%{hovertext}\\u003c\\u002fb\\u003e\\u003cbr\\u003e\\u003cbr\\u003eAttention Patch=%{x}\\u003cbr\\u003eOutput Patch=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"hovertext\":[\"L0H0\",\"L0H1\",\"L0H2\",\"L0H3\",\"L0H4\",\"L0H5\",\"L0H6\",\"L0H7\",\"L0H8\",\"L0H9\",\"L0H10\",\"L0H11\",\"L1H0\",\"L1H1\",\"L1H2\",\"L1H3\",\"L1H4\",\"L1H5\",\"L1H6\",\"L1H7\",\"L1H8\",\"L1H9\",\"L1H10\",\"L1H11\",\"L2H0\",\"L2H1\",\"L2H2\",\"L2H3\",\"L2H4\",\"L2H5\",\"L2H6\",\"L2H7\",\"L2H8\",\"L2H9\",\"L2H10\",\"L2H11\",\"L3H0\",\"L3H1\",\"L3H2\",\"L3H3\",\"L3H4\",\"L3H5\",\"L3H6\",\"L3H7\",\"L3H8\",\"L3H9\",\"L3H10\",\"L3H11\",\"L4H0\",\"L4H1\",\"L4H2\",\"L4H3\",\"L4H4\",\"L4H5\",\"L4H6\",\"L4H7\",\"L4H8\",\"L4H9\",\"L4H10\",\"L4H11\",\"L5H0\",\"L5H1\",\"L5H2\",\"L5H3\",\"L5H4\",\"L5H5\",\"L5H6\",\"L5H7\",\"L5H8\",\"L5H9\",\"L5H10\",\"L5H11\",\"L6H0\",\"L6H1\",\"L6H2\",\"L6H3\",\"L6H4\",\"L6H5\",\"L6H6\",\"L6H7\",\"L6H8\",\"L6H9\",\"L6H10\",\"L6H11\",\"L7H0\",\"L7H1\",\"L7H2\",\"L7H3\",\"L7H4\",\"L7H5\",\"L7H6\",\"L7H7\",\"L7H8\",\"L7H9\",\"L7H10\",\"L7H11\",\"L8H0\",\"L8H1\",\"L8H2\",\"L8H3\",\"L8H4\",\"L8H5\",\"L8H6\",\"L8H7\",\"L8H8\",\"L8H9\",\"L8H10\",\"L8H11\",\"L9H0\",\"L9H1\",\"L9H2\",\"L9H3\",\"L9H4\",\"L9H5\",\"L9H6\",\"L9H7\",\"L9H8\",\"L9H9\",\"L9H10\",\"L9H11\",\"L10H0\",\"L10H1\",\"L10H2\",\"L10H3\",\"L10H4\",\"L10H5\",\"L10H6\",\"L10H7\",\"L10H8\",\"L10H9\",\"L10H10\",\"L10H11\",\"L11H0\",\"L11H1\",\"L11H2\",\"L11H3\",\"L11H4\",\"L11H5\",\"L11H6\",\"L11H7\",\"L11H8\",\"L11H9\",\"L11H10\",\"L11H11\"],\"legendgroup\":\"\",\"marker\":{\"color\":\"#636efa\",\"symbol\":\"circle\"},\"mode\":\"markers\",\"name\":\"\",\"orientation\":\"v\",\"showlegend\":false,\"x\":[0.0006398981786333025,0.005319283809512854,0.0011588380439206958,-5.923740172875114e-05,-0.0010669109178707004,0.005080018658190966,-0.0030812176410108805,-0.0020524668507277966,-0.0014406939735636115,0.003492556745186448,-0.0025678491219878197,-0.0009152094717137516,-0.0007601853576488793,0.0001696908293524757,0.00012250228610355407,-0.00034918184974230826,1.5405081285280176e-05,0.005008597858250141,-0.0002970260684378445,-0.0014443857362493873,-0.001099835499189794,0.00047436909517273307,5.108177356305532e-05,-0.003494771895930171,-0.0007234682561829686,0.0017475537024438381,-0.0001551919267512858,5.749216506956145e-05,-9.699495421955362e-05,-0.00042355581535957754,-0.000791196885984391,0.0002723913639783859,0.00010212998313363642,0.00042187771759927273,0.00015183570212684572,-0.0007432028651237488,0.11458060890436172,0.00021201284835115075,-0.000942025741096586,0.0004294627869967371,0.02004331909120083,0.002104991814121604,7.662265852559358e-05,-0.0015435622772201896,-0.0008476486545987427,-0.0005817683413624763,0.00011934742360608652,-1.9130493456032127e-05,-0.001126819639466703,0.0012377764796838164,-0.0012320708483457565,-0.0005953610525466502,-0.0007538085337728262,-0.0005843526450917125,0.004813097883015871,0.00018163899949286133,-0.0005363250384107232,0.0008580865105614066,-0.00029840212664566934,-1.2183103535789996e-05,-0.004240894690155983,0.0029526741709560156,0.0005220275488682091,0.0009532691328786314,0.0001625084987608716,0.3435087502002716,-0.0003038727736566216,0.00010364028275944293,-0.005300321150571108,0.024866117164492607,0.014383613131940365,-0.0023283488117158413,-0.0023879888467490673,-0.002172317821532488,-0.00047651707427576184,0.0004340272571425885,-0.004673780873417854,0.001858678413555026,-0.0026543056592345238,0.0014364985981956124,0.000303067296044901,0.1304299533367157,8.957769750850275e-05,0.0011777335312217474,0.0003194456803612411,0.020570717751979828,0.00031887509976513684,-0.002512404229491949,-0.0002621884341351688,-0.00024587716325186193,0.0005522335413843393,-0.00043134228326380253,0.0002570534124970436,0.008090788498520851,-0.003069269470870495,-0.0004241599526721984,0.0009768297895789146,0.00039294702582992613,0.0017541655106469989,0.022596266120672226,-4.443644138518721e-05,0.00014159921556711197,0.00958488043397665,-0.0003154182049911469,0.0015266804257407784,0.0011816268088296056,-0.01077402476221323,0.009365785866975784,0.006314304657280445,-0.001094431965611875,0.011661848053336143,0.0013478941982612014,-0.02918611839413643,0.0038340529426932335,-0.04409320279955864,-0.005031823180615902,0.004822427872568369,0.2766473889350891,-3.1280032999347895e-05,-0.0006619150517508388,0.09538879990577698,0.025069067254662514,0.014238725416362286,0.01475451048463583,9.914294059854001e-05,-8.977906691143289e-05,0.0508294515311718,-0.5050979852676392,0.00014717054727952927,-0.0016020277980715036,0.06883121281862259,0.0023271406535059214,0.0013427927624434233,0.009631095454096794,-0.07776330411434174,-0.007728920318186283,-0.0005721024353988469,-0.0029568022582679987,-0.00494667561724782,0.000458796217571944,-0.0006325145368464291,-0.006519974209368229,-0.32048866152763367,-0.002472163178026676],\"xaxis\":\"x\",\"y\":[0.000949342327658087,0.016124119982123375,0.0018556242575868964,0.0034389242064207792,-0.009823138825595379,0.011059673503041267,-0.0040638200007379055,-0.0015785341383889318,-0.001208073808811605,0.003828749991953373,-0.004257441032677889,-0.0011428623693063855,-0.001076644053682685,-0.0003779782564379275,3.0206042538338806e-06,-0.000260510336374864,-0.00014149851631373167,0.0038328110240399837,-0.0004281202855054289,-0.0014291484840214252,-0.0009217541082762182,0.0006938663427717984,0.00043359093251638114,-0.0035708239302039146,-0.0004966544220224023,0.0008070718613453209,0.0005420306115411222,-0.0005315592279657722,-0.0007152119651436806,-0.0010380810126662254,-0.0009492752142250538,-8.538240945199504e-05,0.0002774928288999945,0.0021076432894915342,-0.0001976146304514259,-0.0016398524167016149,0.11626167595386505,0.00025017315056174994,-0.0014669395750388503,-0.0003969745012000203,0.01896214485168457,-0.00018882131553255022,0.011170395649969578,-0.0013304082676768303,-0.0007363561890088022,-0.00030310085276141763,-0.00014579448907170445,-0.0002225178322987631,-0.0016502231592312455,0.00029299859306775033,-0.0014352232683449984,0.030841777101159096,-0.007431894540786743,-0.0002820908557623625,0.006017110776156187,-0.011006947606801987,-0.0012651296565309167,0.001490030437707901,-0.00018016225658357143,0.0029450219590216875,-0.004210856277495623,0.0029608632903546095,0.002045620232820511,0.0013410139363259077,-0.0012189144035801291,0.34349939227104187,0.0005627049831673503,-0.0001257913827430457,-0.005152345169335604,0.016241183504462242,0.017089638859033585,-0.004173669498413801,0.039775215089321136,0.015227536670863628,-0.001023011514917016,0.0008081123232841492,-0.004935599863529205,-0.002123551908880472,-0.014273260720074177,0.0013749118661507964,0.0014838549541309476,0.1302708089351654,-0.00033538774005137384,0.0012914760736748576,0.0003724069392774254,0.0195147804915905,0.00022308839834295213,0.12424899637699127,-0.0004041232750751078,-0.007652163505554199,0.0013012762647122145,-0.0011244030902162194,-0.007449581753462553,0.19224178791046143,-0.0032753082923591137,-0.000501957256346941,-0.001007740618661046,3.1212908652378246e-05,-0.0008586570620536804,0.012360110878944397,-0.00040429108776152134,-0.004328995477408171,0.31855517625808716,0.002329523442313075,0.002118081087246537,0.0001410957775078714,0.27793484926223755,0.005738543812185526,0.0058897752314805984,-0.000968372100032866,0.009125681594014168,0.020675498992204666,-0.03700448200106621,0.014263628050684929,-0.04828385263681412,0.05834108963608742,0.0006516450084745884,0.2636081576347351,0.0004918886115774512,-0.0026110773906111717,0.08374276012182236,0.020676370710134506,-0.0037437702994793653,0.01085111778229475,-0.0010960765648633242,0.0004738992138300091,0.04817944020032883,-0.47990989685058594,0.00018412260396871716,0.011861777864396572,0.06088537722826004,0.0008459034143015742,0.005328110884875059,-0.011493667028844357,-0.11350765824317932,0.006330112461000681,0.0003166264505125582,-0.0011600126745179296,-0.022669030353426933,0.004070901777595282,0.007316238712519407,-0.008345123380422592,-0.27817434072494507,0.003635196480900049],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Attention Patch\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Output Patch\"}},\"legend\":{\"tracegroupgap\":0},\"title\":{\"text\":\"Scatter plot of output patching vs attention patching\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('f93bc8e7-172e-489a-86d5-ed9f1696b43b');\n",
+       "var gd = document.getElementById('fc3a3636-e1a7-4d2b-b7d1-f71b5cc7bbb7');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -14731,37 +14731,37 @@
    "execution_count": 33,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:41:38.079827Z",
-     "iopub.status.busy": "2024-09-05T17:41:38.079652Z",
-     "iopub.status.idle": "2024-09-05T17:41:38.093107Z",
-     "shell.execute_reply": "2024-09-05T17:41:38.092588Z"
+     "iopub.execute_input": "2024-09-10T17:15:17.050075Z",
+     "iopub.status.busy": "2024-09-10T17:15:17.049757Z",
+     "iopub.status.idle": "2024-09-10T17:15:17.063365Z",
+     "shell.execute_reply": "2024-09-10T17:15:17.062821Z"
     }
    },
    "outputs": [
     {
      "data": {
       "text/html": [
-       "<div style='max-width: 700px;'><h2>Top Early Heads</h2><br/><div id=\"circuits-vis-8e46332b-2641\" style=\"margin: 15px 0;\"/>\n",
+       "<div style='max-width: 700px;'><h2>Top Early Heads</h2><br/><div id=\"circuits-vis-49c7fd51-4c0a\" style=\"margin: 15px 0;\"/>\n",
        "    <script crossorigin type=\"module\">\n",
        "    import { render, AttentionHeads } from \"https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js\";\n",
        "    render(\n",
-       "      \"circuits-vis-8e46332b-2641\",\n",
+       "      \"circuits-vis-49c7fd51-4c0a\",\n",
        "      AttentionHeads,\n",
        "      {\"attention\": [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9964697360992432, 0.0035302219912409782, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9966080188751221, 0.0017803364899009466, 0.0016116512706503272, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9577597975730896, 0.002041348023340106, 0.00576710794121027, 0.03443171828985214, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9969283938407898, 0.0006081241299398243, 0.0016115037724375725, 0.0005201429594308138, 0.0003317868977319449, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9679325819015503, 0.0013422606280073524, 0.011834591627120972, 0.004275884013622999, 0.005249680485576391, 0.009365071542561054, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9869164228439331, 0.0013478520559147, 0.008249778300523758, 0.00044454942690208554, 0.0012886020122095942, 0.0008496184600517154, 0.0009031315566971898, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9841058850288391, 0.005145575385540724, 0.0032505656126886606, 0.00023865938419476151, 0.0014756217133253813, 0.000972211651969701, 0.00134052406065166, 0.003471038304269314, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9905135631561279, 0.0026424615643918514, 0.0020855581387877464, 4.1533587591402465e-07, 1.741811320243869e-05, 0.0003096052969340235, 0.00018154106510337442, 0.0010755121475085616, 0.0031737794633954763, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9528716206550598, 0.00732765905559063, 0.008451228961348534, 0.00028046435909345746, 0.002839941531419754, 0.002773231128230691, 0.001109760720282793, 0.0011476994259282947, 0.0057571399956941605, 0.017441321164369583, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04626865312457085, 0.00023600729764439166, 0.00012578870519064367, 0.5745652914047241, 0.3546755015850067, 0.0192879606038332, 9.036171832121909e-05, 0.00044691169750876725, 0.004133324604481459, 3.63319622920244e-06, 0.00016656972002238035, 0.0, 0.0, 0.0, 0.0], [0.887678861618042, 0.005508160684257746, 0.008590048179030418, 0.022789321839809418, 0.034917183220386505, 0.001486042165197432, 7.143727270886302e-05, 0.0006066859932616353, 0.010601978749036789, 0.0001475940807722509, 0.004989828448742628, 0.022612828761339188, 0.0, 0.0, 0.0], [0.9325922131538391, 0.009143135510385036, 0.0023391752038151026, 0.0008593837847001851, 0.0012484431499615312, 0.00037233170587569475, 0.0001396290463162586, 5.667153527610935e-05, 0.029758313670754433, 0.002666445914655924, 0.00340129598043859, 0.0010437102755531669, 0.016379283741116524, 0.0, 0.0], [0.9817627668380737, 0.001917588640935719, 0.00040775517118163407, 1.3564374512498034e-07, 1.264090315089561e-05, 2.9349219403229654e-05, 3.720399035955779e-05, 1.9605669876909815e-05, 0.00034280188265256584, 0.0020146335009485483, 0.007377031724900007, 4.751105734612793e-06, 0.0004324739275034517, 0.005641316995024681, 0.0], [0.908775269985199, 0.004100896418094635, 0.0067780399695038795, 1.580042953719385e-05, 0.00011193934915354475, 8.643363980809227e-05, 3.187262109349831e-06, 0.001585155026987195, 0.003799399361014366, 0.0067048450000584126, 0.0139771094545722, 0.0001160897736554034, 0.004422870930284262, 0.021179812029004097, 0.028343213722109795]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9990116357803345, 0.0009883929742500186, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9985806941986084, 0.0004404807696118951, 0.0009789254982024431, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9970335960388184, 0.0004637633392121643, 0.0009368518949486315, 0.0015657964395359159, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9892035126686096, 8.868379518389702e-05, 0.000574222591239959, 0.009837915189564228, 0.0002956092939712107, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9954046010971069, 0.00010050175478681922, 0.0013415871653705835, 3.886737977154553e-05, 0.0001206161396112293, 0.00299392850138247, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9989855885505676, 0.00010351995297241956, 0.0004172757326159626, 7.227508376672631e-06, 6.046874841558747e-05, 0.0002549678029026836, 0.0001708982017589733, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9980852603912354, 0.00016822919133119285, 0.00010910424316534773, 4.1012435758602805e-06, 5.103566945763305e-05, 0.0001056204637279734, 4.150096719968133e-05, 0.0014351928839460015, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9972643852233887, 7.076081237755716e-05, 6.327207665890455e-05, 2.615450966914068e-08, 5.106949174660258e-06, 3.7383753806352615e-05, 3.455430851317942e-05, 0.000461774121504277, 0.0020626918412745, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9859043955802917, 0.0015692580491304398, 0.001985961804166436, 5.8834184528677724e-06, 0.0003162050561513752, 0.00014976764214225113, 2.4109986043185927e-05, 0.00012667303963098675, 0.0012655562022700906, 0.00865224003791809, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3212882876396179, 0.0026113800704479218, 0.0017525857547298074, 0.48284193873405457, 0.1752953678369522, 0.012053580023348331, 5.7314944569952786e-05, 0.00047417485620826483, 0.0011855909833684564, 4.6230838051997125e-05, 0.0023935972712934017, 0.0, 0.0, 0.0, 0.0], [0.9928818941116333, 0.0011452743783593178, 0.0015605579828843474, 5.570496796281077e-05, 0.0020913302432745695, 0.0002502372954040766, 9.751672678248724e-07, 1.4541505152010359e-05, 0.0006855707615613937, 3.3003976568579674e-05, 0.0003418456471990794, 0.0009391428320668638, 0.0, 0.0, 0.0], [0.9924144148826599, 0.0006524075288325548, 0.00016266616876237094, 7.948347956698854e-06, 0.00015110620006453246, 6.227314588613808e-05, 7.466151714652369e-07, 4.000507090040628e-07, 0.00244068237952888, 3.030487641808577e-05, 4.814808562514372e-05, 9.094910637941211e-05, 0.003938092850148678, 0.0, 0.0], [0.9971888661384583, 0.00062146270647645, 3.946273500332609e-05, 1.9362538239420246e-08, 1.800630002435355e-06, 1.9080694983131252e-05, 9.158955435850658e-06, 7.050003887343337e-07, 6.766093429178e-05, 0.0008072670316323638, 0.00013612494512926787, 2.204007842010469e-06, 2.116413634212222e-05, 0.0010850975522771478, 0.0], [0.9784426689147949, 0.0038412772119045258, 0.001794080133549869, 9.544757631374523e-06, 0.0001613280619494617, 0.00021571738761849701, 3.438441638081713e-07, 0.0001911988656502217, 0.0018583982018753886, 0.002262116875499487, 0.0028807998169213533, 0.00015137717127799988, 3.800494960159995e-05, 0.004419004078954458, 0.0037341148126870394]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9793283343315125, 0.020671749487519264, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.906032145023346, 0.0005559856654144824, 0.09341184794902802, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9916713237762451, 0.0009248568094335496, 0.0019154187757521868, 0.005488428752869368, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9500631093978882, 0.0015093885594978929, 0.00022879372409079224, 0.0011742664501070976, 0.04702451452612877, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9742831587791443, 0.001334858825430274, 0.00016252745990641415, 0.001139267929829657, 1.2687537491729017e-06, 0.023079030215740204, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9734269380569458, 0.0036638062447309494, 0.0007961394148878753, 0.0020336511079221964, 1.7259832020499744e-05, 0.013762583956122398, 0.006299513392150402, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9740443229675293, 0.0018358685774728656, 0.00031735500670038164, 0.0013147101271897554, 4.377634832053445e-05, 0.00045178644359111786, 0.00035931551246903837, 0.021632865071296692, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8537226319313049, 0.0009385455632582307, 1.8928734789369628e-05, 0.0007116434862837195, 4.0574541344540194e-05, 2.706850500544533e-05, 0.0005688024684786797, 0.01303892396390438, 0.1309327930212021, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9718613028526306, 0.0030212444253265858, 0.0018890955252572894, 0.0058778319507837296, 0.0006584360380657017, 0.002494583837687969, 0.001156346406787634, 0.008928273804485798, 0.0013289419002830982, 0.0027839455287903547, 0.0, 0.0, 0.0, 0.0, 0.0], [0.21994593739509583, 0.0028093564324080944, 0.6635686159133911, 0.007326209452003241, 0.001153560820966959, 0.0018521154997870326, 0.0013647314626723528, 0.008370185270905495, 0.0001247532054549083, 0.002670685760676861, 0.09081388264894485, 0.0, 0.0, 0.0, 0.0], [0.9398402571678162, 0.0037398827262222767, 0.0006579139735549688, 0.0027551481034606695, 1.3964868230686989e-05, 0.002081213751807809, 0.007874594070017338, 0.008091994561254978, 0.002785064745694399, 0.003697305917739868, 0.00012857612455263734, 0.028334153816103935, 0.0, 0.0, 0.0], [0.9256278872489929, 0.003680802881717682, 0.0006159098702482879, 0.004310118965804577, 7.914558227639645e-05, 0.0015225791139528155, 0.0014093437930569053, 0.041066594421863556, 0.0011927677551284432, 0.002704742830246687, 8.953329233918339e-05, 0.00031252068583853543, 0.017388159409165382, 0.0, 0.0], [0.7785975933074951, 0.005006679333746433, 8.588898344896734e-05, 0.0016439110040664673, 3.517184086376801e-05, 0.0007985705742612481, 0.004088219255208969, 0.03206512704491615, 0.02768608182668686, 0.003920219838619232, 2.586934351711534e-05, 0.0010157276410609484, 0.009296733886003494, 0.13573415577411652, 0.0], [0.954503059387207, 0.006336927879601717, 0.00032585320877842605, 0.0035448195412755013, 2.0907922589685768e-05, 0.007449527736753225, 0.009912977926433086, 0.005456207785755396, 0.0005529882037080824, 0.0011178080458194017, 6.70458102831617e-05, 0.003978283144533634, 0.0024476239923387766, 6.584440416190773e-05, 0.00422010850161314]]], \"attentionHeadNames\": [\"L5H5\", \"L6H9\", \"L3H0\"], \"tokens\": [\"<|endoftext|>\", \"When\", \" John\", \" and\", \" Mary\", \" went\", \" to\", \" the\", \" shops\", \",\", \" John\", \" gave\", \" the\", \" bag\", \" to\"]}\n",
        "    )\n",
-       "    </script></div><div style='max-width: 700px;'><h2>Top Middle Heads</h2><br/><div id=\"circuits-vis-db6b15e4-7a16\" style=\"margin: 15px 0;\"/>\n",
+       "    </script></div><div style='max-width: 700px;'><h2>Top Middle Heads</h2><br/><div id=\"circuits-vis-c23f4f46-0e8b\" style=\"margin: 15px 0;\"/>\n",
        "    <script crossorigin type=\"module\">\n",
        "    import { render, AttentionHeads } from \"https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js\";\n",
        "    render(\n",
-       "      \"circuits-vis-db6b15e4-7a16\",\n",
+       "      \"circuits-vis-c23f4f46-0e8b\",\n",
        "      AttentionHeads,\n",
        "      {\"attention\": [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9883801937103271, 0.011619755066931248, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9756757020950317, 0.006998097989708185, 0.017326191067695618, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.06910058110952377, 0.005814341828227043, 0.9046230316162109, 0.020462043583393097, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8063386082649231, 0.003319838549941778, 0.0537770576775074, 0.11000825464725494, 0.02655618265271187, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9564918875694275, 0.004251283593475819, 0.011477907188236713, 0.011219605803489685, 0.005006618797779083, 0.011552776210010052, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9077332615852356, 0.0036374430637806654, 0.040214236825704575, 0.012946903705596924, 0.019611867144703865, 0.011057589203119278, 0.004798637703061104, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9552405476570129, 0.009812522679567337, 0.01168749388307333, 0.0090243611484766, 0.0034366166219115257, 0.004399922210723162, 0.0033402289263904095, 0.003058257745578885, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9294586777687073, 0.009998402558267117, 0.004449557047337294, 0.006221562158316374, 0.0014876301866024733, 0.018290331587195396, 0.00712206494063139, 0.0036232024431228638, 0.0193486325442791, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6007111668586731, 0.014475381001830101, 0.09337030351161957, 0.01660209521651268, 0.05098922923207283, 0.10098620504140854, 0.051671840250492096, 0.02743108756840229, 0.02603602036833763, 0.017726697027683258, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6264241933822632, 0.012645086273550987, 0.016070131212472916, 0.024194441735744476, 0.029195599257946014, 0.11857090145349503, 0.06370382010936737, 0.023378603160381317, 0.001658876077271998, 0.06528978794813156, 0.018868643790483475, 0.0, 0.0, 0.0, 0.0], [0.2991042137145996, 0.0021427334286272526, 0.024437544867396355, 0.005155442748218775, 0.010526876896619797, 0.003321276046335697, 0.009254480712115765, 0.005843525752425194, 0.0021181809715926647, 0.017681695520877838, 0.6048213839530945, 0.015592672862112522, 0.0, 0.0, 0.0], [0.8641641139984131, 0.004937537480145693, 0.019023817032575607, 0.001670254161581397, 0.00609729764983058, 0.0005472623743116856, 0.003564003622159362, 0.00030002548010088503, 0.007773987948894501, 0.007786957547068596, 0.06421297788619995, 0.005789309274405241, 0.014132493175566196, 0.0, 0.0], [0.9178580045700073, 0.005138528533279896, 0.003403181442990899, 0.0021888096816837788, 0.002614964498206973, 0.0028868599329143763, 0.0028416835702955723, 0.0013745100004598498, 0.006431094836443663, 0.012306848540902138, 0.0090817641466856, 0.01310056634247303, 0.008110282011330128, 0.012662900611758232, 0.0], [0.019487837329506874, 5.700236215488985e-05, 0.026276404038071632, 0.0009103945340029895, 0.0056328438222408295, 7.981786620803177e-05, 5.7736750022741035e-05, 0.000307683803839609, 0.00014904775889590383, 0.0002852973702829331, 0.9430450201034546, 0.0007767666247673333, 0.0018659740453585982, 0.0006072799442335963, 0.0004609254829119891]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9562382698059082, 0.043761663138866425, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9643059372901917, 0.00742809334769845, 0.028266001492738724, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6456211805343628, 0.029121818020939827, 0.20926059782505035, 0.11599643528461456, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7634043097496033, 0.01714019477367401, 0.026512376964092255, 0.08158667385578156, 0.11135648936033249, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8267454504966736, 0.029910113662481308, 0.015236063860356808, 0.034572526812553406, 0.027030833065509796, 0.06650502979755402, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7882238626480103, 0.02833343856036663, 0.04124196991324425, 0.024983419105410576, 0.05788382515311241, 0.020290205255150795, 0.03904324769973755, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7217290997505188, 0.027771806344389915, 0.02081979066133499, 0.024504197761416435, 0.01813969761133194, 0.014079142361879349, 0.04442400485277176, 0.12853237986564636, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7728042602539062, 0.04133640229701996, 0.006504536606371403, 0.007949033752083778, 0.006013217847794294, 0.01562037505209446, 0.03980163484811783, 0.05545476824045181, 0.054515715688467026, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.47073426842689514, 0.03082239255309105, 0.07406120002269745, 0.055252522230148315, 0.14865727722644806, 0.062336407601833344, 0.034053124487400055, 0.026126855984330177, 0.042438995093107224, 0.05551688000559807, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6962025165557861, 0.0052972170524299145, 0.004229621961712837, 0.05190793052315712, 0.007751153316348791, 0.03517890349030495, 0.03057355433702469, 0.021498825401067734, 0.01566012017428875, 0.051384858787059784, 0.08031534403562546, 0.0, 0.0, 0.0, 0.0], [0.37817656993865967, 0.006224599666893482, 0.023588571697473526, 0.044115372002124786, 0.059451501816511154, 0.00953902117908001, 0.012997719459235668, 0.014349035918712616, 0.0203937329351902, 0.01785479485988617, 0.33459627628326416, 0.07871285080909729, 0.0, 0.0, 0.0], [0.4760020971298218, 0.011026055552065372, 0.018324486911296844, 0.04565680772066116, 0.026791490614414215, 0.011260150000452995, 0.018724240362644196, 0.018448904156684875, 0.04068983346223831, 0.023096060380339622, 0.12447667866945267, 0.08994489908218384, 0.0955582857131958, 0.0, 0.0], [0.7641531229019165, 0.00824385043233633, 0.0032178766559809446, 0.004948926158249378, 0.004036621656268835, 0.00750678451731801, 0.011691190302371979, 0.013843986205756664, 0.01143654901534319, 0.0222361721098423, 0.013085336424410343, 0.03330926224589348, 0.036730363965034485, 0.06555990874767303, 0.0], [0.1303366869688034, 0.0022374859545379877, 0.0373632088303566, 0.06385007500648499, 0.11956273019313812, 0.011286774650216103, 0.006565204821527004, 0.00851503573358059, 0.005633368622511625, 0.007633764296770096, 0.5373668074607849, 0.0301155224442482, 0.020915960893034935, 0.005129658151417971, 0.013487689197063446]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.972596287727356, 0.027403652667999268, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9560405015945435, 0.037302762269973755, 0.006656836252659559, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5009381771087646, 0.0898313894867897, 0.34842997789382935, 0.06080043688416481, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7674180269241333, 0.17525732517242432, 0.021164551377296448, 0.02390032634139061, 0.012259737588465214, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7222559452056885, 0.18388347327709198, 0.023735947906970978, 0.007100591901689768, 0.007225400768220425, 0.05579861253499985, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6940793395042419, 0.13069462776184082, 0.028726812452077866, 0.009216434322297573, 0.011527394875884056, 0.08933128416538239, 0.036424122750759125, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6924421191215515, 0.12459339946508408, 0.01531891617923975, 0.007925059646368027, 0.006713555194437504, 0.0796525627374649, 0.04910353198647499, 0.02425084263086319, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6969695091247559, 0.10134164988994598, 0.011634294874966145, 0.007826745510101318, 0.0032780379988253117, 0.042647555470466614, 0.05842007324099541, 0.042913682758808136, 0.03496843948960304, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6869087815284729, 0.023936893790960312, 0.10650163143873215, 0.02047020196914673, 0.02552187815308571, 0.04722552374005318, 0.022190820425748825, 0.007246002089232206, 0.019754212349653244, 0.04024401307106018, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5810369849205017, 0.011352411471307278, 0.0033976146951317787, 0.0365644209086895, 0.020549381151795387, 0.051692016422748566, 0.024651870131492615, 0.015468521043658257, 0.049509111791849136, 0.15767459571361542, 0.04810298979282379, 0.0, 0.0, 0.0, 0.0], [0.43488532304763794, 0.006897220853716135, 0.012041947804391384, 0.003814741736277938, 0.010545267723500729, 0.0076634688302874565, 0.007842864841222763, 0.006417531985789537, 0.044214341789484024, 0.10391034185886383, 0.3332052230834961, 0.028561806306242943, 0.0, 0.0, 0.0], [0.4747830927371979, 0.024993136525154114, 0.009554888121783733, 0.0037503365892916918, 0.008160687983036041, 0.02339756116271019, 0.010678932070732117, 0.0014935131184756756, 0.030863573774695396, 0.19498929381370544, 0.15084564685821533, 0.054150503128767014, 0.012338940985500813, 0.0, 0.0], [0.7144870758056641, 0.009456266649067402, 0.0035061524249613285, 0.0029858576599508524, 0.001229096669703722, 0.01199608389288187, 0.008696629665791988, 0.007586767431348562, 0.009631272405385971, 0.06210443004965782, 0.014414231292903423, 0.07928328961133957, 0.04383841156959534, 0.030784491449594498, 0.0], [0.1013382077217102, 0.0038837045431137085, 0.01792380027472973, 0.0038827748503535986, 0.009177407249808311, 0.01316938828676939, 0.0034797503612935543, 0.0019711176864802837, 0.004929723683744669, 0.1296602189540863, 0.6630275249481201, 0.0330243818461895, 0.0089839156717062, 0.0017172213410958648, 0.003830986563116312]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.97651207447052, 0.023487908765673637, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9743125438690186, 0.009410635568201542, 0.016276877373456955, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5441411137580872, 0.019679147750139236, 0.33595189452171326, 0.10022785514593124, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8594159483909607, 0.021821973845362663, 0.025710705667734146, 0.03842515870928764, 0.05462610721588135, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7619735598564148, 0.07210072875022888, 0.037441689521074295, 0.04506158083677292, 0.031744565814733505, 0.05167779326438904, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8025401830673218, 0.026160430163145065, 0.033131979405879974, 0.02643769420683384, 0.03139319270849228, 0.04932845011353493, 0.031008044257760048, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7639790773391724, 0.021559985354542732, 0.016495496034622192, 0.025408243760466576, 0.027263421565294266, 0.06810054928064346, 0.047422830015420914, 0.029770376160740852, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7543362379074097, 0.01597309671342373, 0.008247380144894123, 0.00967292208224535, 0.013097813352942467, 0.03411843255162239, 0.03947027027606964, 0.032298993319272995, 0.09278487414121628, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5856387615203857, 0.0213167741894722, 0.026237597689032555, 0.020694918930530548, 0.02824069745838642, 0.11104536801576614, 0.047170888632535934, 0.022986045107245445, 0.04925474151968956, 0.08741410821676254, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6466761231422424, 0.015195527113974094, 0.00594041682779789, 0.02527434751391411, 0.010863534174859524, 0.08686516433954239, 0.032730914652347565, 0.021324681118130684, 0.04200880602002144, 0.09539096057415009, 0.017729468643665314, 0.0, 0.0, 0.0, 0.0], [0.42771559953689575, 0.014421668834984303, 0.024129344150424004, 0.026094987988471985, 0.01488838903605938, 0.010527816601097584, 0.012212568894028664, 0.015342887490987778, 0.017334001138806343, 0.04439026489853859, 0.28982794284820557, 0.10311443358659744, 0.0, 0.0, 0.0], [0.4405166506767273, 0.018903780728578568, 0.010929570533335209, 0.015466002747416496, 0.013489533215761185, 0.020115477964282036, 0.019655633717775345, 0.005925851874053478, 0.022772811353206635, 0.06206439808011055, 0.16126078367233276, 0.17319339513778687, 0.03570614382624626, 0.0, 0.0], [0.7304301261901855, 0.008421944454312325, 0.0022352896630764008, 0.004027598071843386, 0.0036414540372788906, 0.009633221663534641, 0.0213489830493927, 0.021966980770230293, 0.02610298991203308, 0.05093422904610634, 0.0083627263084054, 0.023242846131324768, 0.038515783846378326, 0.051135752350091934, 0.0], [0.16577893495559692, 0.004480816423892975, 0.0077864062041044235, 0.007284588646143675, 0.01337976660579443, 0.019251490011811256, 0.010158197022974491, 0.01172470673918724, 0.011870641261339188, 0.034697193652391434, 0.1448296457529068, 0.4143996834754944, 0.08938297629356384, 0.019558662548661232, 0.04541636258363724]]], \"attentionHeadNames\": [\"L8H6\", \"L8H10\", \"L7H9\", \"L7H3\"], \"tokens\": [\"<|endoftext|>\", \"When\", \" John\", \" and\", \" Mary\", \" went\", \" to\", \" the\", \" shops\", \",\", \" John\", \" gave\", \" the\", \" bag\", \" to\"]}\n",
        "    )\n",
-       "    </script></div><div style='max-width: 700px;'><h2>Top Late Heads</h2><br/><div id=\"circuits-vis-7ba2561c-b325\" style=\"margin: 15px 0;\"/>\n",
+       "    </script></div><div style='max-width: 700px;'><h2>Top Late Heads</h2><br/><div id=\"circuits-vis-9c2de0cf-406e\" style=\"margin: 15px 0;\"/>\n",
        "    <script crossorigin type=\"module\">\n",
        "    import { render, AttentionHeads } from \"https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js\";\n",
        "    render(\n",
-       "      \"circuits-vis-7ba2561c-b325\",\n",
+       "      \"circuits-vis-9c2de0cf-406e\",\n",
        "      AttentionHeads,\n",
        "      {\"attention\": [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9628916382789612, 0.03710832819342613, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9778335094451904, 0.0034864130429923534, 0.018680110573768616, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8737697601318359, 0.005964328069239855, 0.08126737922430038, 0.03899851813912392, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8745138645172119, 0.017276111990213394, 0.018874678760766983, 0.054268334060907364, 0.035066962242126465, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9704693555831909, 0.0035177830141037703, 0.0003239834331907332, 0.004373463336378336, 0.0003953034174628556, 0.02092011272907257, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7694658041000366, 0.006685878615826368, 0.011966485530138016, 0.040966276079416275, 0.03344443440437317, 0.04401002824306488, 0.0934610366821289, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8177722692489624, 0.009112595580518246, 0.0042336746118962765, 0.03900325670838356, 0.012653318233788013, 0.007337852846831083, 0.07410021126270294, 0.0357869453728199, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8309933543205261, 0.018796611577272415, 0.00330327614210546, 0.01983213610947132, 0.004009497817605734, 0.012078307569026947, 0.03246007487177849, 0.010322270914912224, 0.06820447742938995, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.31403300166130066, 0.01522789802402258, 0.19789263606071472, 0.059379592537879944, 0.3236042559146881, 0.009037697687745094, 0.019917670637369156, 0.012475269846618176, 0.013939663767814636, 0.03449229896068573, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7319568991661072, 0.03087025322020054, 0.005064091179519892, 0.08703826367855072, 0.004982746671885252, 0.044725559651851654, 0.02350614219903946, 0.0063977413810789585, 0.0034696452785283327, 0.048653144389390945, 0.013335512951016426, 0.0, 0.0, 0.0, 0.0], [0.13710583746433258, 0.012017901986837387, 0.14592327177524567, 0.015683427453041077, 0.44125232100486755, 0.004342416767030954, 0.011117888614535332, 0.007466601673513651, 0.0012341630645096302, 0.01946023479104042, 0.16842849552631378, 0.035967420786619186, 0.0, 0.0, 0.0], [0.4052484929561615, 0.009720955975353718, 0.03158125653862953, 0.09247038513422012, 0.048808369785547256, 0.008174203336238861, 0.031708307564258575, 0.02787904627621174, 0.14128026366233826, 0.0302340779453516, 0.03392859548330307, 0.03171432018280029, 0.10725171118974686, 0.0, 0.0], [0.808469295501709, 0.0056702690199017525, 0.006755992770195007, 0.008696584962308407, 0.010473440401256084, 0.003464736510068178, 0.006760997697710991, 0.0025889205280691385, 0.015097705647349358, 0.01603602059185505, 0.011288060806691647, 0.017277495935559273, 0.00779919745400548, 0.07962135970592499, 0.0], [0.02939111366868019, 0.0016060526249930263, 0.06005561351776123, 0.008175727911293507, 0.8099164366722107, 0.00039796155760996044, 0.0028282543644309044, 0.0033632616978138685, 0.0001569116284372285, 0.0036459483671933413, 0.05494074895977974, 0.001666039228439331, 0.015368940308690071, 5.016689101466909e-05, 0.008436810225248337]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9124024510383606, 0.08759760111570358, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.97407466173172, 0.01434413529932499, 0.011581187136471272, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8240247964859009, 0.03700513392686844, 0.06643965095281601, 0.07253041118383408, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5370795726776123, 0.15483324229717255, 0.0903128981590271, 0.12131187319755554, 0.0964624434709549, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9388002157211304, 0.028733234852552414, 0.0003431019140407443, 0.010481251403689384, 0.000839930260553956, 0.02080223336815834, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6318807005882263, 0.059302832931280136, 0.03517911583185196, 0.0317409411072731, 0.1174689382314682, 0.06869616359472275, 0.05573126673698425, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7625216841697693, 0.04435169696807861, 0.010552966967225075, 0.034276288002729416, 0.014203979633748531, 0.012703591957688332, 0.07259687036275864, 0.048792947083711624, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5725991725921631, 0.12681783735752106, 0.03727060556411743, 0.0431414395570755, 0.04129580408334732, 0.046447426080703735, 0.029164383187890053, 0.03751189261674881, 0.06575141847133636, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.12316936254501343, 0.026347072795033455, 0.27960509061813354, 0.041989635676145554, 0.44105827808380127, 0.006138973869383335, 0.007793003693223, 0.007175281643867493, 0.011286810971796513, 0.055436521768569946, 0.0, 0.0, 0.0, 0.0, 0.0], [0.44855940341949463, 0.1432485729455948, 0.010774216614663601, 0.07855041325092316, 0.01404787041246891, 0.014864671975374222, 0.008633698336780071, 0.0030146560166031122, 0.012792127206921577, 0.16408587992191315, 0.10142850130796432, 0.0, 0.0, 0.0, 0.0], [0.10228201746940613, 0.0238832775503397, 0.055250268429517746, 0.0520680733025074, 0.5759841799736023, 0.0011948152678087354, 0.0013662497512996197, 0.0025677781086415052, 0.0018699930515140295, 0.043160032480955124, 0.10887196660041809, 0.03150142729282379, 0.0, 0.0, 0.0], [0.39313194155693054, 0.04406267777085304, 0.04009911045432091, 0.07407406717538834, 0.04061643034219742, 0.007346798665821552, 0.009507199749350548, 0.013865168206393719, 0.0640539675951004, 0.04894229769706726, 0.0627753809094429, 0.0999651551246643, 0.10155977308750153, 0.0, 0.0], [0.7573198676109314, 0.015095721930265427, 0.007350177504122257, 0.013156400062143803, 0.005726401228457689, 0.008556416258215904, 0.008282607421278954, 0.010307075455784798, 0.002904881490394473, 0.017329057678580284, 0.015188980847597122, 0.0796574279665947, 0.030244464054703712, 0.02888045459985733, 0.0], [0.08579524606466293, 0.023773644119501114, 0.028759371489286423, 0.050303805619478226, 0.674644947052002, 0.0010763936443254352, 0.0011514866491779685, 0.002721975091844797, 0.0005617713322862983, 0.039091579616069794, 0.04861969128251076, 0.015520088374614716, 0.0040845321491360664, 0.00043291584006510675, 0.023462524637579918]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9979304075241089, 0.002069620881229639, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9971864819526672, 0.0010516609763726592, 0.0017618348356336355, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9594857096672058, 0.001310725579969585, 0.03694308176636696, 0.002260456094518304, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9891613125801086, 0.001052629784680903, 0.0048541901633143425, 0.001104357186704874, 0.003827564185485244, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9963598847389221, 0.0007981774397194386, 0.0007764195324853063, 0.00019262291607446969, 0.0002416159404674545, 0.0016313291853293777, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.981286346912384, 0.001863775309175253, 0.006344134453684092, 0.0002960922720376402, 0.004604941233992577, 0.0013966941041871905, 0.00420788861811161, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9950011372566223, 0.0013230532640591264, 0.0007920759962871671, 0.00024644460063427687, 0.0003405151073820889, 0.00016826140927150846, 0.0002837753272615373, 0.0018447580514475703, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9920291900634766, 0.0013539871433749795, 0.0007168339216150343, 9.214139572577551e-05, 0.00013419234892353415, 0.00019706363673321903, 0.00035278062568977475, 0.0002426079590804875, 0.004881155211478472, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5483196377754211, 0.0075838202610611916, 0.28265973925590515, 0.0029685653280466795, 0.14815661311149597, 0.0008172960951924324, 0.0006479909643530846, 0.0017078499076887965, 0.004129356704652309, 0.003009046893566847, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8642559051513672, 0.0036493062507361174, 0.004925842396914959, 0.06957440823316574, 0.038683053106069565, 0.009609837085008621, 0.0008548588375560939, 0.0005643004551529884, 0.003707138355821371, 0.0013952680164948106, 0.002780066104605794, 0.0, 0.0, 0.0, 0.0], [0.30252957344055176, 0.006584456656128168, 0.1401754468679428, 0.030065739527344704, 0.4806697964668274, 0.0005923578282818198, 0.0005309724947437644, 0.0024134721606969833, 0.008505754172801971, 0.0013766245683655143, 0.023764921352267265, 0.0027908117044717073, 0.0, 0.0, 0.0], [0.7773900032043457, 0.003722917754203081, 0.022378109395503998, 0.015139483846724033, 0.017408354207873344, 0.002416662173345685, 0.000709561922121793, 0.0007399603491649032, 0.13850137591362, 0.0023606533650308847, 0.005776867736130953, 0.0028014343697577715, 0.010654616169631481, 0.0, 0.0], [0.9735962152481079, 0.0012812522472813725, 0.002617918187752366, 9.891873924061656e-05, 0.0005098494002595544, 0.00012000725837424397, 0.00045229491661302745, 0.00010779645526781678, 0.002954537281766534, 0.001644388772547245, 0.0012141242623329163, 0.00018637391622178257, 0.0008510759216733277, 0.01436527632176876, 0.0], [0.0990300253033638, 0.000971083587501198, 0.06413238495588303, 0.005246965680271387, 0.8110400438308716, 8.250321843661368e-05, 6.596777529921383e-05, 0.0011991349747404456, 0.0008852760074660182, 0.00024891181965358555, 0.010336406528949738, 7.952339365147054e-05, 0.0049433172680437565, 0.00028918671887367964, 0.0014492359478026628]]], \"attentionHeadNames\": [\"L10H7\", \"L11H10\", \"L9H9\"], \"tokens\": [\"<|endoftext|>\", \"When\", \" John\", \" and\", \" Mary\", \" went\", \" to\", \" the\", \" shops\", \",\", \" John\", \" gave\", \" the\", \" bag\", \" to\"]}\n",
        "    )\n",
@@ -14876,10 +14876,10 @@
    "execution_count": 34,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:41:38.095280Z",
-     "iopub.status.busy": "2024-09-05T17:41:38.094960Z",
-     "iopub.status.idle": "2024-09-05T17:41:38.253988Z",
-     "shell.execute_reply": "2024-09-05T17:41:38.253313Z"
+     "iopub.execute_input": "2024-09-10T17:15:17.065644Z",
+     "iopub.status.busy": "2024-09-10T17:15:17.065330Z",
+     "iopub.status.idle": "2024-09-10T17:15:17.221647Z",
+     "shell.execute_reply": "2024-09-10T17:15:17.220971Z"
     }
    },
    "outputs": [],
@@ -14898,21 +14898,21 @@
    "execution_count": 35,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:41:38.256840Z",
-     "iopub.status.busy": "2024-09-05T17:41:38.256442Z",
-     "iopub.status.idle": "2024-09-05T17:41:38.265714Z",
-     "shell.execute_reply": "2024-09-05T17:41:38.265179Z"
+     "iopub.execute_input": "2024-09-10T17:15:17.224442Z",
+     "iopub.status.busy": "2024-09-10T17:15:17.224020Z",
+     "iopub.status.idle": "2024-09-10T17:15:17.234273Z",
+     "shell.execute_reply": "2024-09-10T17:15:17.233691Z"
     }
    },
    "outputs": [
     {
      "data": {
       "text/html": [
-       "<div style='max-width: 800px;'><h2>Induction Heads</h2><br/><div id=\"circuits-vis-4f5c8cd3-096b\" style=\"margin: 15px 0;\"/>\n",
+       "<div style='max-width: 800px;'><h2>Induction Heads</h2><br/><div id=\"circuits-vis-3078e839-768c\" style=\"margin: 15px 0;\"/>\n",
        "    <script crossorigin type=\"module\">\n",
        "    import { render, AttentionHeads } from \"https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js\";\n",
        "    render(\n",
-       "      \"circuits-vis-4f5c8cd3-096b\",\n",
+       "      \"circuits-vis-3078e839-768c\",\n",
        "      AttentionHeads,\n",
        "      {\"attention\": [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9995203018188477, 0.0004796621506102383, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9989193677902222, 0.00011719338363036513, 0.0009634991874918342, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.999607503414154, 4.1751700337044895e-05, 9.65828076004982e-06, 0.00034111665445379913, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9989573955535889, 3.091409962507896e-05, 1.212368533742847e-05, 0.0002459374663885683, 0.0007535542827099562, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9994303584098816, 3.2271585951093584e-05, 2.162250348192174e-05, 0.000438225019024685, 4.1958148358389735e-05, 3.5627250326797366e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.998076319694519, 1.6859989045769908e-05, 2.043985659838654e-06, 6.719817611156031e-05, 1.8109081167949626e-07, 0.00016430420510005206, 0.0016730953939259052, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9989305138587952, 0.0001340252347290516, 1.5985064237611368e-05, 0.000491619051899761, 6.565183241491468e-08, 6.484094228653703e-06, 1.1354626622051e-05, 0.00040989238186739385, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9949487447738647, 0.0005971402861177921, 1.4574082342733163e-05, 0.0018354525091126561, 9.743422424435266e-07, 7.463527435902506e-05, 5.5863079069240484e-06, 0.00020289771782699972, 0.002319993684068322, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9855782985687256, 6.499102164525539e-05, 8.181475277524441e-05, 0.0024699484929442406, 4.049685031759509e-08, 3.498060686979443e-05, 0.001560866367071867, 0.0018861292628571391, 0.00011740217451006174, 0.008205565623939037, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9950633645057678, 0.00021360788377933204, 0.00014613209350500256, 0.00030726491240784526, 2.292262615810614e-05, 0.0002385093830525875, 1.9861789041897282e-05, 0.000960799225140363, 0.0001365413481835276, 0.0016751259099692106, 0.001215968164615333, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9907191395759583, 0.00013381309690885246, 3.014638878084952e-06, 0.0023974282667040825, 2.1563739210250787e-05, 0.0004241704009473324, 1.383451308356598e-05, 5.452497134683654e-05, 2.246501207991969e-05, 0.0017353435978293419, 0.0022565408144146204, 0.002218238776549697, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.99763023853302, 7.048921543173492e-05, 1.2420621260389453e-06, 6.923218461452052e-05, 0.0007476441096514463, 0.000763593299780041, 7.526726676587714e-06, 7.549879228463396e-05, 2.769195930341084e-07, 9.996434528147802e-05, 3.433594974922016e-05, 0.00019309873459860682, 0.00030683254590258, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9413580298423767, 0.0009730295860208571, 0.0004518784990068525, 0.0012102690525352955, 2.08603341889102e-06, 0.0018085278570652008, 1.965607407328207e-05, 0.03966164216399193, 0.0004176551301497966, 0.003035214263945818, 0.002604098990559578, 0.00038386619416996837, 0.007335478439927101, 0.0007384858326986432, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.98450767993927, 0.00010836195724550635, 2.4642587959533557e-05, 0.0004206285811960697, 1.2694325732809375e-06, 0.00011915803042938933, 9.773763304110616e-05, 0.0023661903105676174, 0.00036190610262565315, 0.003964339382946491, 0.0009553474374115467, 0.0006011889781802893, 0.002696628449484706, 0.00010731287329690531, 0.003667621174827218, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9236841797828674, 0.0004220438713673502, 1.1906979125342332e-05, 0.036959629505872726, 0.00012378330575302243, 0.0005373558960855007, 0.0029662158340215683, 0.0006545133655890822, 5.279582546791062e-05, 0.0077215940691530704, 0.0131848668679595, 7.147376891225576e-05, 0.00951047521084547, 0.0007938834605738521, 0.0006112216506153345, 0.0026941129472106695, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9707465171813965, 7.250557973748073e-05, 4.7150632553893956e-07, 0.0007667056052014232, 1.613832296243345e-06, 4.375730350147933e-05, 0.00013073488662485033, 0.0010341654997318983, 0.000541800691280514, 0.006641003768891096, 0.014592697843909264, 0.00018458777049090713, 0.003935785032808781, 3.716452511071111e-06, 2.8841026505688205e-05, 0.0008475544163957238, 0.0004274799139238894, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9573933482170105, 0.00019599858205765486, 2.3228804479913379e-07, 0.003389731515198946, 9.590152103555738e-07, 5.782760854344815e-05, 1.3954500900581479e-05, 9.882948506856337e-05, 0.00014849765284452587, 0.006945967674255371, 0.01562657207250595, 8.081036867224611e-06, 0.011182066984474659, 6.349627255985979e-06, 2.5934892619261518e-05, 0.00012999530008528382, 0.0007364018238149583, 0.004039338324218988, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.990619421005249, 6.466358172474429e-05, 9.901318662741687e-07, 0.00021274133177939802, 1.4796115465287585e-05, 0.00017918922821991146, 6.778116585337557e-06, 7.372434629360214e-05, 0.0002996290859300643, 0.0013087284751236439, 0.0005674119456671178, 0.0011630650842562318, 0.0028649545274674892, 9.635516471462324e-05, 8.433945913566276e-05, 0.00028768807533197105, 4.3501222535269335e-05, 0.00038704575854353607, 0.001724953530356288, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9956457018852234, 2.669214882189408e-05, 3.9460044121142346e-08, 0.001181368250399828, 1.471557879995089e-05, 0.0001445170200895518, 0.0004370114766061306, 3.213761010556482e-05, 8.180093402643251e-08, 9.59495737333782e-05, 0.0007029082044027746, 1.2536466783785727e-05, 0.00018710187578108162, 0.00011147692566737533, 7.006971713963139e-07, 4.411066583998036e-06, 0.0007388183148577809, 4.973216618964216e-06, 9.905266779242083e-05, 0.0005597113631665707, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9956768155097961, 3.464311521383934e-05, 5.037085770709382e-08, 0.00012908628559671342, 3.5288087474327767e-06, 8.07559808890801e-06, 3.11900585074909e-05, 6.715198833262548e-05, 1.5350008197856368e-07, 0.00010576295608188957, 0.0004207944730296731, 4.739598443848081e-05, 0.00017379986820742488, 3.910295163223054e-06, 3.7129705106053734e-06, 6.050187221262604e-05, 4.920005812891759e-05, 4.588661977322772e-05, 0.0002989978529512882, 0.00027882878202944994, 0.0025604318361729383, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9873839616775513, 0.002735754009336233, 1.0963850400003139e-05, 0.0005862272810190916, 6.101510280132061e-06, 6.327711162157357e-05, 2.1911782823735848e-05, 0.00033786968560889363, 6.588418182218447e-05, 0.0013113167369738221, 0.0019715256057679653, 0.00013529015996027738, 0.0018900128779932857, 0.00018075927800964564, 0.00026106255245395005, 0.00037814932875335217, 9.485983173362911e-05, 4.87527868244797e-05, 0.00035683103487826884, 0.0007243757136166096, 0.00010285121243214235, 0.0013322114245966077, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3808109164237976, 0.005228960886597633, 0.6019217371940613, 0.006041430402547121, 3.194508099113591e-05, 4.269973578630015e-05, 8.645826892461628e-05, 0.0006069993833079934, 0.0006299586966633797, 0.0009580003097653389, 0.0006117315497249365, 0.0002489396429155022, 0.0002464656427036971, 6.495700654340908e-05, 0.0006897134589962661, 0.00021990980894770473, 7.002806523814797e-05, 0.00014192514936439693, 2.700363074836787e-05, 5.169477663002908e-05, 4.5723249968432356e-06, 0.00029897037893533707, 0.0009650306892581284, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.563797116279602, 0.0085763493552804, 0.004927160684019327, 0.34193748235702515, 2.1189502149354666e-05, 0.00020753320131916553, 0.0003593186556827277, 0.0014688574010506272, 0.0003070282400585711, 0.02934207208454609, 0.004261575639247894, 8.383052772842348e-06, 0.0024304059334099293, 9.930188389262185e-05, 0.000542300462257117, 9.130241960519925e-05, 0.03218064829707146, 7.990232552401721e-05, 0.0003810952475760132, 0.00010837644367711619, 3.33801144734025e-05, 0.0008215161506086588, 0.0017809885321184993, 0.006236794404685497, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3740951418876648, 0.00012492266250774264, 8.869202247296926e-06, 0.0009476605337113142, 0.6135826706886292, 0.002516972366720438, 0.0002441502292640507, 5.642483665724285e-05, 1.795958496586536e-06, 0.0004439418844413012, 5.296186645864509e-05, 0.0001609584578545764, 0.0007701200665906072, 0.005056836176663637, 1.382048776576994e-05, 5.882297773496248e-05, 0.00018946161435451359, 0.00012655068712774664, 3.178522456437349e-05, 6.542137998621911e-05, 1.7726270016282797e-05, 0.0008768205880187452, 0.00038724290789105, 9.015519140120887e-07, 0.00016804093320388347, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.18932704627513885, 0.0003522285260260105, 1.1454378181952052e-05, 0.0007539974758401513, 0.003070831298828125, 0.7761431336402893, 5.7844576076604426e-05, 0.004086362663656473, 1.6754691387177445e-05, 0.0018927170895040035, 0.00031636765925213695, 0.0005395743064582348, 0.005218177568167448, 0.007962456904351711, 0.004062105901539326, 0.002489584032446146, 1.4082329471420962e-05, 1.4990718227636535e-05, 0.00015581185289192945, 0.0003996991435997188, 5.5086038628360257e-05, 0.0019400393357500434, 0.0003281844255980104, 1.4123389746600878e-07, 8.962309038906824e-06, 0.0007822647457942367, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.02010822296142578, 2.635493547131773e-05, 2.7333503567206208e-06, 0.0002452948538120836, 1.4769997505936772e-05, 2.0045878045493737e-05, 0.9765581488609314, 0.0007374248816631734, 2.2332008029479766e-06, 0.00016837129078339785, 0.00137846521101892, 1.0426949756947579e-06, 9.273762407246977e-05, 6.001193469273858e-06, 1.852532113844063e-05, 7.490423013223335e-05, 2.892227712436579e-05, 4.0283404814545065e-05, 1.959171459020581e-05, 2.944982952612918e-05, 8.47621322463965e-06, 0.0002561432193033397, 5.941176277701743e-05, 2.014576523379219e-07, 7.743915921309963e-05, 7.494620604120428e-06, 1.74276083271252e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.22277945280075073, 0.002593980636447668, 2.9011016522417776e-06, 0.0005522919236682355, 1.904411533359962e-06, 0.0006727795698679984, 0.0021306686103343964, 0.6761497855186462, 6.977441080380231e-05, 0.003776289988309145, 0.06293372064828873, 0.0015865935711190104, 0.0017082190606743097, 3.881497832480818e-05, 0.009209989570081234, 0.0013166061835363507, 0.00018038915004581213, 0.001267515355721116, 0.0018657379550859332, 0.0009925103513523936, 3.7589063140330836e-05, 0.006426853593438864, 0.0020985312294214964, 3.3772406027310353e-07, 2.868737283279188e-05, 2.698123182653944e-07, 0.00025180247030220926, 0.001325962133705616, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08230321109294891, 0.0005252842092886567, 4.260616606188705e-06, 0.0006264003459364176, 8.583574384601889e-08, 4.224356871418422e-06, 4.716531293524895e-06, 0.0003961432375945151, 0.8586453795433044, 0.05151975154876709, 0.0026339711621403694, 1.6450698240078054e-05, 0.0006709631998091936, 1.5013217762316344e-06, 9.040276927407831e-05, 7.098149944795296e-05, 7.727313459326979e-06, 0.00020618800772354007, 0.0005295966984704137, 0.0013012847630307078, 5.2623075674773645e-08, 9.972530097002164e-05, 0.00018814747454598546, 2.380612386332359e-06, 6.255433254409581e-05, 3.8025717685741256e-08, 9.557845714880386e-07, 3.836729320028098e-06, 8.390715811401606e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.465234637260437, 0.004093279130756855, 6.743345238646725e-06, 0.004121774807572365, 5.443537247629138e-07, 5.419518129201606e-05, 9.294981850871409e-07, 0.00026017383788712323, 0.00907624140381813, 0.5008078813552856, 0.0023183668963611126, 2.4238510377472267e-05, 0.0030431346967816353, 4.68218931928277e-05, 0.00012087374489055946, 0.0001290314394282177, 0.00011785242531914264, 4.014383375761099e-05, 0.0024619202595204115, 0.0034230302553623915, 1.7263497511521564e-06, 0.0002791083534248173, 0.002281592693179846, 4.889936462859623e-06, 0.00025089550763368607, 2.8178769184705743e-07, 1.050695664162049e-05, 1.070987195816997e-06, 4.786956196767278e-05, 0.0017402973026037216, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3585144281387329, 0.0011027655564248562, 0.00016592990141361952, 0.013357667252421379, 2.546785253798589e-07, 6.185192614793777e-05, 0.0057292357087135315, 0.010611956007778645, 0.0007551729213446379, 0.06400889158248901, 0.4949103593826294, 8.391502342419699e-05, 0.005665580276399851, 2.122150362993125e-06, 0.0002814042672980577, 0.0004684626474045217, 0.0004158185620326549, 0.0012447141343727708, 0.012713258154690266, 0.0010433174902573228, 1.1126050594612025e-05, 0.0011420734226703644, 0.0006368785980157554, 7.042537617962807e-05, 0.002744977129623294, 1.2284903050385765e-07, 1.483806499891216e-05, 0.005110186990350485, 0.004603618290275335, 0.00015696643094997853, 0.014371593482792377, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.387024849653244, 0.0012950673699378967, 4.0595266909804195e-05, 0.0016053735744208097, 5.915556903346442e-05, 0.00023957251687534153, 2.72913839580724e-05, 0.004568422678858042, 0.0004761815653182566, 0.01562541350722313, 0.011085761711001396, 0.49522680044174194, 0.03870179131627083, 0.00026717083528637886, 0.0020083452109247446, 0.004042898770421743, 0.00032641590223647654, 0.0036371152382344007, 0.005066386889666319, 0.003996914252638817, 5.871349458175246e-06, 0.012405160814523697, 0.004702437669038773, 1.1574948075576685e-05, 0.0001442672364646569, 1.3832131116942037e-05, 7.560355879832059e-05, 1.5469962818315253e-05, 0.0020267819054424763, 3.810078851529397e-05, 0.003570300992578268, 0.001669161138124764, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4725484848022461, 0.0012542733456939459, 1.6021392639231635e-06, 0.007470476441085339, 2.098127151839435e-05, 0.00022495862504001707, 3.144747097394429e-06, 0.00015600249753333628, 5.6978915381478146e-05, 0.008746420964598656, 0.006501034367829561, 0.005835692398250103, 0.4674169719219208, 0.0013900346821174026, 0.0022286404855549335, 0.0015983363846316934, 0.0009148887475021183, 4.253075894666836e-05, 0.007089768536388874, 0.002937783719971776, 9.027125088323373e-06, 0.002526575466617942, 0.004196966998279095, 9.68642325460678e-07, 0.0005582970916293561, 1.858499126683455e-05, 8.944844012148678e-05, 6.846757969469763e-06, 7.68072932260111e-05, 1.4904732779541519e-05, 0.0030944147147238255, 0.0010494303423911333, 0.0019187491852790117, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14304865896701813, 0.0001688210031716153, 5.145543582329992e-07, 0.00013525491522159427, 0.001670985366217792, 0.000564366695471108, 4.049839844810776e-06, 4.1035786125576124e-05, 6.743645712958823e-07, 0.00026636559050530195, 3.54247895302251e-05, 0.00038403936196118593, 0.001104785711504519, 0.8484564423561096, 0.00022315295063890517, 0.00125994929112494, 2.182349817303475e-05, 4.0234138509731565e-07, 9.149295692623127e-06, 7.982308306964114e-05, 3.274861228419468e-05, 0.000818326196167618, 0.0002738100884016603, 6.036240307594198e-08, 5.658039754052879e-06, 0.0007130159065127373, 0.00026146037271246314, 3.5881919302482856e-06, 2.1928895876044407e-05, 4.159101507639207e-08, 0.00011863742111017928, 1.6391287545047817e-06, 3.8121448596939445e-05, 0.00023530120961368084, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.24328209459781647, 0.0009063901379704475, 0.0002717962197493762, 0.0012521950993686914, 3.105668383796001e-06, 0.001761557999998331, 7.013605227257358e-06, 0.040556132793426514, 0.0003538632008712739, 0.003045480465516448, 0.0016025315271690488, 0.0006016580737195909, 0.009119357913732529, 0.00086904939962551, 0.6830158829689026, 0.002626276109367609, 0.00010224885045317933, 7.45571151128388e-06, 3.991548510384746e-05, 0.00021531074889935553, 2.0861371012870222e-06, 0.000726094760466367, 0.0004009866388514638, 1.702951885818038e-05, 6.652241427218542e-05, 3.073251377827546e-07, 0.00018005711899604648, 5.3525491239270195e-06, 0.0071047646924853325, 2.1172911147004925e-05, 0.00044749100925400853, 5.941993367741816e-05, 3.588023173506372e-05, 0.001178023056127131, 0.00011552044452400878, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.24815866351127625, 0.0004487273108679801, 9.697681889520027e-06, 0.0010719134006649256, 7.626599654031452e-06, 0.00018865316815208644, 6.339164974633604e-05, 0.004560624714940786, 0.0010741765145212412, 0.015649856999516487, 0.002170345513150096, 0.0007331715896725655, 0.007097125519067049, 0.00044510516454465687, 0.016780385747551918, 0.6907860040664673, 0.00019302917644381523, 0.0003835348761640489, 0.0009248864953406155, 0.001862714416347444, 2.057407328948102e-07, 0.002524268114939332, 0.0015590769471600652, 3.3248329600610305e-07, 1.708475247141905e-05, 4.2712878212114447e-07, 1.1026169886463322e-05, 4.378564426588127e-06, 0.00015843719302210957, 1.5476502085221e-05, 0.0009950557723641396, 4.288491982151754e-05, 5.0623875722521916e-05, 0.000995215610601008, 1.3690095329366159e-05, 0.0010022438364103436, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15202470123767853, 0.0005997071275487542, 1.1835468285426032e-05, 0.10644718259572983, 9.087667422136292e-05, 0.0003228001296520233, 0.000542075838893652, 0.0006674103206023574, 0.00022456956503447145, 0.021209707483649254, 0.019741186872124672, 0.00012132841220591217, 0.015023781917989254, 0.0003238526696804911, 0.0003435326216276735, 0.003281805431470275, 0.6119738221168518, 0.00224235188215971, 0.008034387603402138, 0.0039560976438224316, 0.00143277901224792, 0.011238718405365944, 0.0009119117166846991, 1.6121168300742283e-05, 0.034571193158626556, 4.049971539643593e-05, 7.184680725913495e-05, 0.0005501421983353794, 0.0001563764235470444, 6.361229679896496e-06, 0.00147312565241009, 0.0009954210836440325, 1.2639467058761511e-05, 0.0010144341504201293, 3.702372487168759e-05, 5.788439739262685e-05, 0.00023058420629240572, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.035443101078271866, 2.0004812540719286e-05, 4.18805292667912e-08, 0.0003809734480455518, 5.874024395779998e-07, 2.792186933220364e-06, 2.425247657811269e-05, 8.103247819235548e-05, 0.00011144909512950107, 0.0027193769346922636, 0.00454584788531065, 4.083846943103708e-05, 0.0014098533429205418, 5.68071186535235e-07, 3.1567922178510344e-06, 8.66913323989138e-05, 0.0001365105708828196, 0.8487799763679504, 0.10322029143571854, 0.00040226749842986465, 2.244725919808843e-06, 0.0014588433550670743, 0.00012918819265905768, 6.671142216418957e-08, 7.290130452020094e-05, 3.444819185460801e-07, 1.4172115925248363e-06, 1.9379569494049065e-05, 4.5301294449018314e-05, 6.376938017638167e-06, 0.00026465157861821353, 0.0004582676920108497, 2.116967152687721e-05, 9.94484216789715e-05, 4.348984106172793e-08, 3.564729809113487e-07, 2.450852434776607e-06, 8.022138899832498e-06, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2318733036518097, 0.0003618353803176433, 8.543166529761947e-08, 0.006350396666675806, 1.0386469284640043e-06, 1.706758121144958e-05, 9.30641544982791e-06, 0.00014515469956677407, 9.0070599981118e-05, 0.01544187217950821, 0.04194100946187973, 8.23421487439191e-06, 0.022686535492539406, 5.4958513828751165e-06, 2.1405727238743566e-05, 5.323981531546451e-05, 0.0014441021485254169, 0.005718633532524109, 0.6479307413101196, 0.012128104455769062, 4.028283728985116e-05, 0.002520335605368018, 0.0007621925324201584, 6.570355992607801e-08, 0.0007668299949727952, 4.547823095890635e-07, 8.124479791149497e-06, 1.3828406736138277e-05, 3.526886212057434e-05, 9.165401934296824e-06, 0.0020918850786983967, 0.004286687355488539, 3.2024597658164566e-06, 0.0017294595018029213, 5.357091481528187e-07, 2.501844164726208e-06, 2.884956984416931e-06, 8.676738070789725e-05, 0.0014117915416136384, 0.0, 0.0, 0.0, 0.0], [0.3010721206665039, 0.0003067184588871896, 7.214547395051341e-07, 0.0010787947103381157, 3.671133526950143e-05, 9.958079317584634e-05, 2.7117800982523477e-06, 8.51582590257749e-05, 0.0006431526271626353, 0.00647402461618185, 0.0024339223746210337, 0.0026007171254605055, 0.012783723883330822, 0.0001047167825163342, 7.190534233814105e-05, 0.00023829020210541785, 0.00014132891374174505, 0.0010901358909904957, 0.006501613184809685, 0.6443015933036804, 0.0005654482520185411, 0.011171936988830566, 0.0017330633709207177, 3.266845851612743e-07, 7.267465116456151e-05, 1.726137998048216e-05, 5.252150003798306e-05, 2.8104516331950435e-06, 2.3941935069160536e-05, 6.633036537095904e-05, 0.002350717782974243, 0.00014581516734324396, 0.0007032529101707041, 0.0013972821179777384, 3.202367588528432e-05, 2.3009477445157245e-05, 2.68677049461985e-05, 1.7945263607543893e-05, 0.0001209545589517802, 0.001408109557814896, 0.0, 0.0, 0.0], [0.012865342199802399, 1.5182659808488097e-05, 2.972013524171757e-09, 0.0002875621721614152, 1.2528023489721818e-06, 1.252613128599478e-05, 4.250307028996758e-06, 8.040903594519477e-06, 3.2277103656497275e-08, 5.239708480075933e-05, 0.00013598549412563443, 1.002539192995755e-06, 5.5890355724841356e-05, 7.012691639829427e-06, 9.724845284608818e-08, 3.212419130704802e-07, 4.670393173000775e-05, 1.5471712231374113e-06, 1.7591864889254794e-05, 5.334270463208668e-05, 0.9805435538291931, 0.005806764122098684, 1.860846532508731e-05, 1.2037257857144823e-09, 3.0451696147792973e-05, 6.754650598850276e-07, 4.56516090707737e-06, 5.748487637902144e-06, 3.3398507639503805e-06, 7.534131096775809e-10, 7.689136509725358e-06, 3.901714080711827e-06, 4.307142731363456e-08, 5.192545700083429e-07, 5.106750791128434e-07, 8.534786211100709e-09, 1.0071753209217604e-08, 2.59315993389464e-06, 1.4836371065030107e-07, 3.5096545047963446e-07, 4.402238118927926e-06, 0.0, 0.0], [0.2408866286277771, 0.0003843706217594445, 3.8272382596460375e-08, 0.0007521198713220656, 4.429959517437965e-06, 1.3616883734357543e-05, 1.0370951713412069e-05, 0.00026089049060828984, 2.180447154387366e-06, 0.0008239771705120802, 0.0025784452445805073, 0.0001127499490394257, 0.001231508213095367, 3.884355010086438e-06, 3.183517401339486e-05, 0.00015905841428320855, 0.00016030347615014762, 0.00039731874130666256, 0.0017629768699407578, 0.001543086371384561, 0.0018108984222635627, 0.7393378019332886, 0.00752194132655859, 2.4824957645108725e-09, 2.4742143068579026e-05, 3.882949215494591e-07, 2.5054800971702207e-06, 3.208101816198905e-06, 4.296221959521063e-05, 2.8116325268001674e-08, 1.3948349078418687e-05, 1.6366040654247627e-05, 6.464315902121598e-06, 3.004602695000358e-05, 1.3071932869479497e-07, 9.608824029783136e-07, 1.787801124919497e-06, 1.753078095134697e-06, 4.386996351968264e-06, 1.6937889085966162e-05, 1.7175851098727435e-05, 2.5725426894496195e-05, 0.0], [0.5535919070243835, 0.17262260615825653, 5.0000453484244645e-05, 0.01042777020484209, 8.65629335748963e-05, 0.0002456592337694019, 5.955337837804109e-05, 0.001045247190631926, 0.0008384220418520272, 0.01357327215373516, 0.011254074051976204, 0.003105845768004656, 0.02095966413617134, 0.0010894862934947014, 0.0004093680181540549, 0.002362574217841029, 0.0016563499812036753, 0.0009357953676953912, 0.0033135719131678343, 0.00543051166459918, 0.0010150577872991562, 0.032745711505413055, 0.1570950448513031, 3.374255175003782e-05, 0.0007222488638944924, 2.940950980701018e-05, 6.13709562458098e-05, 3.093114719376899e-05, 0.00024038334959186614, 4.459061528905295e-05, 0.0006923413602635264, 0.00030414367211051285, 0.0001727685157675296, 0.0012000406859442592, 0.00010696896788431332, 2.8368263883749023e-05, 1.842521487560589e-05, 2.8223597837495618e-05, 3.744996865862049e-05, 0.00011934195208596066, 0.00018961371097248048, 3.498619480524212e-05, 0.0019905013032257557]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9940896034240723, 0.0059104301035404205, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9918997287750244, 0.002507372060790658, 0.005592965055257082, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.99346524477005, 0.003521271515637636, 0.00027059370768256485, 0.0027428686153143644, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9289718270301819, 0.003740765620023012, 0.0001203682622872293, 0.002464901190251112, 0.06470214575529099, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9885430335998535, 0.0006141006015241146, 0.00041145377326756716, 0.007392834406346083, 0.000539508939255029, 0.002499047899618745, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9694589376449585, 0.0009395341039635241, 2.8261547413421795e-05, 0.0020353952422738075, 2.71175304078497e-05, 0.00829475186765194, 0.019215956330299377, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9474747776985168, 0.005424594972282648, 0.004983103834092617, 0.021659083664417267, 1.2853307453042362e-05, 0.0011076473165303469, 0.001525420811958611, 0.01781250163912773, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9157270193099976, 0.006453890353441238, 0.0011921703116968274, 0.04077358543872833, 3.2812098652357236e-05, 0.002033841796219349, 0.00022136607731226832, 0.010999627411365509, 0.022565681487321854, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8299553990364075, 0.0021487579215317965, 0.002185288118198514, 0.04418088495731354, 1.725433139654342e-05, 0.0014081649715080857, 0.020800573751330376, 0.08762412518262863, 0.000955707801040262, 0.010723880492150784, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9581490159034729, 0.004779844544827938, 0.00035515165654942393, 0.0029918320942670107, 0.0032907268032431602, 0.007958430796861649, 0.0017856257036328316, 0.013284464366734028, 0.00015784945571795106, 0.0009225388057529926, 0.0063244919292628765, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.944205641746521, 0.0028841965831816196, 4.829783574678004e-05, 0.007261567283421755, 0.000468916492536664, 0.005381070543080568, 0.00017496546206530184, 0.006957130506634712, 0.00012532716209534556, 0.004074090626090765, 0.016885550692677498, 0.011533298529684544, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7170465588569641, 0.0029263142496347427, 1.372535371046979e-05, 0.0010318169370293617, 0.1216491237282753, 0.14445394277572632, 0.0009355274378322065, 0.005332366097718477, 4.7773132791917305e-06, 0.0012480594450607896, 0.00030511102522723377, 0.0019744164310395718, 0.0030781622044742107, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7894357442855835, 0.0020973754581063986, 0.0020318119786679745, 0.010099953971803188, 0.00036911203642375767, 0.06713488698005676, 0.001530290232039988, 0.08991599082946777, 0.0002695393341127783, 0.005345543846487999, 0.005062697920948267, 0.0006220968207344413, 0.019640257582068443, 0.006444734986871481, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8822957873344421, 0.0018603845965117216, 0.00040732327033765614, 0.003660863032564521, 0.0005736579769290984, 0.007988521829247475, 0.02257135882973671, 0.02860121801495552, 0.0003292130131740123, 0.0042566233314573765, 0.006161138415336609, 0.0010443199425935745, 0.02063308097422123, 0.005851183086633682, 0.013765350915491581, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.786965548992157, 0.0042131864465773106, 0.0001677500840742141, 0.06983140110969543, 0.002818087814375758, 0.007715716026723385, 0.031199535354971886, 0.017051437869668007, 0.00018332427134737372, 0.012005697004497051, 0.017662225291132927, 0.0002890100877266377, 0.03032497689127922, 0.004889386706054211, 0.003095801919698715, 0.011587009765207767, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9286779761314392, 0.0015812769997864962, 1.4040829228179064e-06, 0.0019428395899012685, 1.1035229363187682e-05, 0.0002707966777961701, 0.002038027858361602, 0.0036648765671998262, 0.00022354313114192337, 0.0007879888289608061, 0.04505370557308197, 0.00048093145596794784, 0.010849039070308208, 3.6126039049122483e-06, 6.594638398382813e-05, 0.0008642339380457997, 0.0034828479401767254, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8384721875190735, 0.0026208001654595137, 3.342468335176818e-05, 0.01232653483748436, 7.991644088178873e-05, 0.003431107848882675, 0.0022304044105112553, 0.008115601725876331, 0.0006596868042834103, 0.008109694346785545, 0.0638236477971077, 0.00041881605284288526, 0.03681996464729309, 0.00012104758206987754, 0.0005988333723507822, 0.00862804614007473, 0.003624672070145607, 0.00988560076802969, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8221591114997864, 0.0057971584610641, 0.00019727385370060802, 0.005305740050971508, 0.0009490191005170345, 0.01202884316444397, 0.00190554594155401, 0.0023991470225155354, 0.01059637125581503, 0.014539752155542374, 0.011935291811823845, 0.01170417945832014, 0.048201464116573334, 0.006271726917475462, 0.006691561546176672, 0.02466079778969288, 0.005656811874359846, 0.005013483576476574, 0.003986795898526907, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9537661671638489, 0.0015298646176233888, 5.0040102905768435e-06, 0.007204053923487663, 0.0002875779173336923, 0.0017434227047488093, 0.00982952956110239, 0.004322257358580828, 3.13615760205721e-06, 0.00010985424887621775, 0.008465062826871872, 0.0003484692715574056, 0.0013010905822739005, 0.0008953196229413152, 6.606405804632232e-05, 5.8244528190698475e-05, 0.0037666442804038525, 0.0005685480427928269, 0.001456835656426847, 0.004272699821740389, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9650986790657043, 0.0014334715669974685, 4.691014055424603e-06, 0.0012125533539801836, 0.0003355584340170026, 0.0012382488930597901, 0.0013702923897653818, 0.002564979949966073, 6.760245128134557e-07, 7.471856224583462e-05, 0.001875573769211769, 0.0005754472804255784, 0.0028297884855419397, 0.0007055316236801445, 0.00027714675525203347, 0.00048612672253511846, 0.001569450250826776, 0.0003992257115896791, 0.0028444859199225903, 0.004788658116012812, 0.010314569808542728, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8593109250068665, 0.020267227664589882, 0.002777141286060214, 0.013781089335680008, 0.00030035056988708675, 0.002284612273797393, 0.0010426462395116687, 0.009922247380018234, 0.003133647609502077, 0.007885495200753212, 0.009642801247537136, 0.001854671980254352, 0.00787472166121006, 0.0011870941380038857, 0.002824161434546113, 0.00242280843667686, 0.004221721086651087, 0.004099525045603514, 0.006750837899744511, 0.0058750067837536335, 0.002170956926420331, 0.030370241031050682, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10544665902853012, 0.01270485483109951, 0.7639007568359375, 0.08842571824789047, 0.00018797894881572574, 0.0002628951915539801, 0.0007833988056518137, 0.001709726988337934, 0.002409537322819233, 0.014530741609632969, 0.0006739442469552159, 3.582143835956231e-05, 0.0008068872266449034, 0.000150876454426907, 0.001663832925260067, 0.0002909536415245384, 0.001969669247046113, 2.807584314723499e-05, 8.648991206428036e-05, 0.0001831684639910236, 5.4813881433801726e-05, 0.0004074028111062944, 0.003285784274339676, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.33122679591178894, 0.007398602087050676, 0.006046590860933065, 0.38090354204177856, 0.0007559282821603119, 0.003547786269336939, 0.00850890576839447, 0.02200973406434059, 0.0006401475402526557, 0.06486588716506958, 0.012087974697351456, 2.1607689632219262e-05, 0.004766723141074181, 0.0004923217929899693, 0.002488807076588273, 0.00041253515519201756, 0.11648410558700562, 0.00017437727365177125, 0.001129980431869626, 0.0012968714581802487, 0.001893762033432722, 0.006374640390276909, 0.012117794714868069, 0.014354660175740719, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.022601181641221046, 0.0004386410873848945, 2.7166800009581493e-06, 0.0003694462648127228, 0.9364854693412781, 0.032159216701984406, 0.00016640119429212064, 0.00037730683106929064, 2.719356189118116e-07, 9.796216181712225e-05, 2.4832312192302197e-05, 0.00014765629020985216, 0.0004301169828977436, 0.004391680471599102, 0.0003911138919647783, 5.064404831500724e-05, 0.00018704244575928897, 6.401351129170507e-06, 2.0583942387020215e-05, 0.0002504437288735062, 4.058926060679369e-05, 0.0006710129673592746, 0.0006265917327255011, 2.0016514099552296e-06, 6.063750697649084e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.039748288691043854, 0.0016230319160968065, 2.9864083899155958e-06, 0.0004959744983352721, 0.0075991530902683735, 0.903511106967926, 0.00030855785007588565, 0.020541049540042877, 3.7260215322021395e-06, 0.0009059454896487296, 0.0005234951386228204, 0.00019036592857446522, 0.002769604790955782, 0.001758625265210867, 0.006682183127850294, 0.0016490903217345476, 0.00027723185485228896, 2.3619986677658744e-05, 0.0003351525228936225, 0.001790589652955532, 0.00015660120698157698, 0.0019276170060038567, 0.0019798888824880123, 1.2426404509824351e-06, 5.812747258460149e-05, 0.005136688239872456, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.017394619062542915, 8.34300517453812e-05, 1.299515815844643e-06, 0.0006728374864906073, 1.4733542229805607e-05, 0.00010638036474119872, 0.9528148770332336, 0.021972037851810455, 7.827152899153589e-07, 1.5493780665565282e-05, 0.0031969614792615175, 3.257389471400529e-05, 0.0001382468908559531, 2.389342444075737e-05, 0.000840842432808131, 0.00010894631850533187, 0.00028938756440766156, 0.0001089509969460778, 6.79441072861664e-05, 0.00012526210048235953, 0.00019726519531104714, 0.0008631302043795586, 0.0002980587596539408, 2.3374896045424975e-06, 0.0003466096241027117, 2.3777269234415144e-05, 0.00025927400565706193, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05558102950453758, 0.0006445589242503047, 4.0934705225481594e-07, 0.00042194841080345213, 1.340792550763581e-06, 0.000646521570160985, 0.0018358062952756882, 0.9107965230941772, 6.180139735079138e-06, 0.00014045907300896943, 0.010802187956869602, 0.0002091953792842105, 0.0011495650978758931, 1.2848360711359419e-05, 0.003265448845922947, 0.0001199487887788564, 0.00020614646200556308, 0.0006146047380752861, 0.0012190378038212657, 0.001267205341719091, 0.00038551114266738296, 0.0035668814089149237, 0.0011672672117128968, 5.425858375929238e-07, 0.0003176873433403671, 1.1132586905659991e-06, 0.002803834853693843, 0.002816123189404607, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08360803127288818, 0.0049411761574447155, 5.052056076237932e-05, 0.007228298112750053, 1.1925421858904883e-06, 0.000162318188813515, 0.0002728418621700257, 0.005935708060860634, 0.6047206521034241, 0.26288917660713196, 0.005585616920143366, 7.11743050487712e-05, 0.0013994898181408644, 2.673692279131501e-06, 0.0015110736712813377, 9.4614464615006e-05, 0.0003633210726547986, 0.0007099778158590198, 0.0009741897229105234, 0.003578288946300745, 1.0970221410389058e-05, 0.0001563943224027753, 0.003689345670863986, 0.0002846125280484557, 0.004334022291004658, 1.3892353081246256e-06, 0.00024127897631842643, 0.0001852042623795569, 0.006996475625783205, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1152101382613182, 0.006714800372719765, 0.000162112366524525, 0.0255129411816597, 6.206613761605695e-06, 0.0004051269206684083, 5.4560387070523575e-05, 0.00351078063249588, 0.014202400110661983, 0.7775508165359497, 0.0034905581269413233, 5.7819983339868486e-05, 0.0047684297896921635, 3.606372411013581e-05, 0.0007714288076385856, 0.0003429505741223693, 0.002843409776687622, 9.518005390418693e-05, 0.003136677900329232, 0.005728919059038162, 4.441967757884413e-05, 0.0004939687205478549, 0.008516553789377213, 0.0005445042625069618, 0.014522203244268894, 5.894724836252863e-06, 0.00030799367232248187, 2.679031968000345e-05, 0.0010097086196765304, 0.009926472790539265, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.03713580593466759, 0.0008775312453508377, 7.848548557376489e-05, 0.015299011021852493, 1.4544053783538402e-06, 0.00010877755266847089, 0.006165174767374992, 0.014768626540899277, 7.875503069953993e-05, 0.001974995480850339, 0.845594584941864, 7.480713975382969e-05, 0.0028255584184080362, 5.115834369462391e-07, 0.00017026669229380786, 0.00015173685096669942, 0.006004323251545429, 0.0034732050262391567, 0.012612894177436829, 0.0006110360845923424, 0.0005419608787633479, 0.0009975193534046412, 0.002906821435317397, 0.0003956863656640053, 0.020268796011805534, 1.613207473383227e-06, 0.00021117184951435775, 0.007802305277436972, 0.017504550516605377, 2.8824499167967588e-05, 0.0013330691726878285, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04171165078878403, 0.0007273957598954439, 5.084281838207971e-06, 0.0004205020668450743, 0.0002682959020603448, 0.00036437364178709686, 0.00021653271687682718, 0.0014350030105561018, 5.140026132721687e-06, 9.150908590527251e-05, 0.0012519678566604853, 0.9164962768554688, 0.025602256879210472, 0.0013075522147119045, 0.0030348259024322033, 0.0005533328512683511, 0.00034456001594662666, 0.00023525662254542112, 0.0004293296951800585, 0.00137053313665092, 3.670702062663622e-05, 0.0015910869697108865, 0.00048218751908279955, 9.56441908783745e-06, 0.00015524028276558965, 0.00017542645218782127, 0.0004897234030067921, 6.953987031010911e-05, 0.0008855744963511825, 1.1304659892630298e-06, 4.837113374378532e-05, 0.0001840855402406305, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1554105579853058, 0.0020603404846042395, 5.904707904846873e-06, 0.005144172348082066, 0.00025823267060332, 0.0014501075493171811, 5.391828017309308e-05, 0.0021640402264893055, 2.1527968783630058e-05, 0.0016064351657405496, 0.010191709734499454, 0.010151085443794727, 0.7491754293441772, 0.0027421838603913784, 0.01017574779689312, 0.005842445883899927, 0.0030403982382267714, 3.083394767600112e-05, 0.006112964358180761, 0.016732651740312576, 0.00015879959391895682, 0.0020674674306064844, 0.0016374929109588265, 1.1047049156331923e-05, 0.0014025868149474263, 0.0001992537872865796, 0.001220017671585083, 2.1487703634193167e-05, 0.0007244196604005992, 5.2953896556573454e-06, 0.0011675384594127536, 0.0018881573341786861, 0.0071258689276874065, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.03857047110795975, 0.00048676287406124175, 8.812001084379517e-08, 0.000122779980301857, 0.005581730045378208, 0.0025055643636733294, 2.1938227291684598e-05, 0.00011670660751406103, 8.340171575582644e-08, 6.456951086875051e-05, 1.648088073125109e-05, 0.00033052131766453385, 0.0009664044482633471, 0.9410394430160522, 0.0025494787842035294, 0.000986628350801766, 0.00023048926959745586, 2.1702271624235436e-07, 9.28598819882609e-06, 0.001075997482985258, 8.328325202455744e-05, 0.0006203338853083551, 0.0003860929573420435, 1.7099183935442852e-07, 1.0056071005237754e-05, 0.0019989400170743465, 0.0020207997877150774, 4.052772510476643e-06, 1.9309491108288057e-05, 3.980539009518225e-09, 1.3065056009509135e-05, 4.567590394799481e-07, 5.0416052545187995e-05, 0.00011722340423148125, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.32133859395980835, 0.0015032502124086022, 0.00035717309219762683, 0.009410788305103779, 0.00016890434199012816, 0.016028327867388725, 0.0002053897042060271, 0.019911326467990875, 6.106824002927169e-05, 0.0025995566975325346, 0.0018900673603639007, 0.0003402142319828272, 0.018113745376467705, 0.0032540573738515377, 0.5562437772750854, 0.008188020437955856, 0.0034578992053866386, 2.759976268862374e-05, 0.00021329266019165516, 0.004111619666218758, 7.46081059332937e-05, 0.0024103508330881596, 0.0031445701606571674, 0.0002873913326766342, 0.0028005591593682766, 5.076818706584163e-05, 0.010558384470641613, 8.335074380738661e-05, 0.007083627860993147, 1.715210783004295e-05, 0.0007566043641418219, 0.00013426884834188968, 4.393113704281859e-05, 0.0036519141867756844, 0.0014778183540329337, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.16036191582679749, 0.0012780018150806427, 2.2185913621797226e-05, 0.0023713421542197466, 0.0002248762029921636, 0.001783479005098343, 0.003998146392405033, 0.00463333074003458, 4.07644038205035e-05, 0.0015416644746437669, 0.0015637290198355913, 0.00047258863924071193, 0.01683974452316761, 0.0034527594689279795, 0.017767908051609993, 0.7616898417472839, 0.004626939073204994, 0.00039177873986773193, 0.000968358654063195, 0.0050940741784870625, 3.4372260415693745e-06, 0.0016141458181664348, 0.0016999210929498076, 1.1049841305066366e-05, 0.00029084301786497235, 4.1652379877632484e-05, 0.000830529781524092, 0.00043556030141189694, 0.0004473649896681309, 1.989761585718952e-06, 0.0003503866319078952, 6.375126395141706e-05, 6.234755710465834e-05, 0.0025966004468500614, 0.0005663724150508642, 0.00186053360812366, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.16436104476451874, 0.00263788178563118, 4.14170281146653e-05, 0.06607695668935776, 0.0011643714969977736, 0.0015624640509486198, 0.01255388930439949, 0.0027841543778777122, 3.6274232115829363e-05, 0.004333572927862406, 0.006344497669488192, 0.00013702016440220177, 0.022220438346266747, 0.001139097730629146, 0.0015757762594148517, 0.003894534194841981, 0.5750948190689087, 0.0019101364305242896, 0.004216517321765423, 0.005103054456412792, 0.0011178136337548494, 0.011998940259218216, 0.006141388788819313, 8.207879727706313e-05, 0.0696118101477623, 0.0019772748928517103, 0.0024057570844888687, 0.01315633486956358, 0.0013538515195250511, 9.255067197955213e-06, 0.003937660250812769, 0.0014234132831916213, 5.669038000633009e-05, 0.005460998509079218, 0.0004025489615742117, 0.0006155924638733268, 0.003060702932998538, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.07928115874528885, 0.0006841905415058136, 6.815574948859648e-08, 0.0010951507138088346, 2.0406362182257e-06, 2.2378419089363888e-05, 0.0006314062047749758, 0.0003747308801393956, 2.3558166503789835e-05, 0.00018500992155168205, 0.008331519551575184, 7.647382153663784e-05, 0.00418297341093421, 2.444711526550236e-07, 1.1477874068077654e-05, 5.1371527661103755e-05, 0.0018863201839849353, 0.6889944076538086, 0.1946372091770172, 0.004766413476318121, 4.9866932386066765e-05, 0.0028069806285202503, 0.005558052100241184, 3.95493913174505e-07, 0.0008065987494774163, 3.353459078425658e-06, 5.2719624363817275e-05, 0.0003899818693753332, 0.0004888054681941867, 7.204272606031736e-06, 9.927168866852298e-05, 0.003168433904647827, 3.9500540879089385e-05, 0.0009883888997137547, 7.518784883586704e-08, 1.4315326097857906e-06, 6.484955520136282e-06, 0.00029431298025883734, 0.0, 0.0, 0.0, 0.0, 0.0], [0.28373828530311584, 0.004487877711653709, 5.058388069301145e-06, 0.019317252561450005, 5.4460702813230455e-05, 0.0012376051163300872, 0.0017604499589651823, 0.0063126953318715096, 0.00024137768195942044, 0.006789366248995066, 0.07160750776529312, 0.0003089959791395813, 0.06864359229803085, 4.332327443989925e-05, 0.0004192555497866124, 0.0020601244177669287, 0.006795847322791815, 0.01733478344976902, 0.33858922123908997, 0.03910181671380997, 0.0007660706178285182, 0.009993083775043488, 0.007829555310308933, 1.3115719411871396e-05, 0.01356492843478918, 6.270289304666221e-05, 0.0021037741098552942, 0.0009936662390828133, 0.00333457812666893, 9.826785390032455e-05, 0.006462769117206335, 0.03356613591313362, 0.00019831513054668903, 0.030609816312789917, 1.5713876564404927e-05, 0.0001204574218718335, 0.0005064337747171521, 0.001992761390283704, 0.018918951973319054, 0.0, 0.0, 0.0, 0.0], [0.25413063168525696, 0.007289606146514416, 1.6370242519769818e-05, 0.005644946824759245, 0.00047911194269545376, 0.003032122040167451, 0.0008039328386075795, 0.0005737085011787713, 0.001770934322848916, 0.0052279396913945675, 0.006573305930942297, 0.0037488588131964207, 0.039559945464134216, 0.0013069461565464735, 0.002049593720585108, 0.0032922588288784027, 0.005070534534752369, 0.0043929279781877995, 0.0029604455921798944, 0.5652295351028442, 0.006637280806899071, 0.020123612135648727, 0.013719636015594006, 3.509566158754751e-05, 0.00249256263487041, 0.0006804370786994696, 0.0050042420625686646, 0.0004963083192706108, 0.0002992783847730607, 0.0005683296476490796, 0.006156585179269314, 0.0014163292944431305, 0.002080241683870554, 0.01701081357896328, 0.0011587258195504546, 0.0008447651634924114, 0.0012247745180502534, 0.002057015197351575, 0.0027713908348232508, 0.0020689249504357576, 0.0, 0.0, 0.0], [0.02277885377407074, 0.00018681293295230716, 1.6159448890107342e-08, 0.0006753625930286944, 3.9720243876217864e-06, 1.8965230992762372e-05, 0.00014574530359823257, 0.00015363575948867947, 1.437089469646935e-08, 3.714960257639177e-06, 0.00038176553789526224, 9.189574484480545e-06, 0.00013344796025194228, 1.550033084640745e-05, 4.871678356721532e-06, 4.94858227284567e-07, 0.0004079081991221756, 1.881936623249203e-05, 3.495846613077447e-05, 0.00018379271205049008, 0.8935850262641907, 0.08001779019832611, 0.0004949963185936213, 6.95558668439844e-08, 0.0003611761494539678, 4.126275143789826e-06, 4.2135678086197004e-05, 0.0001535058399895206, 9.319555101683363e-05, 8.534826179129595e-10, 9.503900741947291e-07, 2.6511430405662395e-05, 9.136761605077481e-07, 4.42588907390018e-06, 3.6229419038136257e-06, 4.963176820638182e-07, 6.715119571509831e-09, 3.286409264546819e-05, 3.7280919968907256e-06, 3.7352272101998096e-06, 1.2850606253778096e-05, 0.0, 0.0], [0.04306096211075783, 0.00043120133341290057, 4.85736819655358e-08, 0.00029497192008420825, 6.446816769312136e-06, 1.794057789084036e-05, 5.2368359320098534e-05, 0.00012123115448048338, 8.686286356862638e-09, 4.085900854988722e-06, 0.00020099191169720143, 2.1525078409467824e-05, 0.0004682287108153105, 1.1544256267370656e-05, 3.74926021322608e-05, 7.720605935901403e-06, 0.0005387069541029632, 8.118282858049497e-05, 0.0002466407313477248, 0.0005759781342931092, 0.0023978566750884056, 0.9488213062286377, 0.002120599150657654, 3.640034407226267e-08, 7.682813156861812e-05, 2.893220880650915e-06, 1.444738245481858e-05, 1.2731605238514021e-05, 2.7991360184387304e-05, 3.7476780012646316e-10, 5.714333610740141e-07, 1.0850835678866133e-05, 3.7292174965841696e-06, 4.0900900785345584e-05, 3.826700321951648e-06, 6.126327207311988e-06, 5.169480346012278e-07, 4.086086119059473e-05, 1.2315524145378731e-05, 2.700609366002027e-05, 3.1881831091595814e-05, 0.00016747090558055788, 0.0], [0.26228708028793335, 0.04815318435430527, 0.0006689673755317926, 0.021233247593045235, 9.166418021777645e-05, 0.00039222234045155346, 0.00020396779291331768, 0.0021344097331166267, 0.001094849780201912, 0.004910758230835199, 0.006146768573671579, 0.0005123682785779238, 0.009844929911196232, 0.0001430303673259914, 0.0010762682650238276, 0.00034581031650304794, 0.010050452314317226, 0.008435730822384357, 0.008105281740427017, 0.00602466706186533, 0.0038135156501084566, 0.04830698296427727, 0.49913153052330017, 0.0012536782305687666, 0.033902086317539215, 0.00010517781629459932, 0.00037178199272602797, 8.200502634281293e-05, 0.0012060013832524419, 0.0001819609315134585, 0.0013505015522241592, 0.0009115547291003168, 0.00011081639240728691, 0.0016774724936112761, 3.677561107906513e-05, 0.00017677698633633554, 4.57889873359818e-05, 0.00101330759935081, 0.0026200786232948303, 0.002004811307415366, 0.0010430817492306232, 0.00036673821159638464, 0.008431942202150822]]], \"attentionHeadNames\": [\"L6H9\", \"L5H5\"], \"tokens\": [\"<|endoftext|>\", \"Research\", \" in\", \" mechan\", \"istic\", \" interpret\", \"ability\", \" seeks\", \" to\", \" explain\", \" behaviors\", \" of\", \" machine\", \" learning\", \" models\", \" in\", \" terms\", \" of\", \" their\", \" internal\", \" components\", \".\", \"Research\", \" in\", \" mechan\", \"istic\", \" interpret\", \"ability\", \" seeks\", \" to\", \" explain\", \" behaviors\", \" of\", \" machine\", \" learning\", \" models\", \" in\", \" terms\", \" of\", \" their\", \" internal\", \" components\", \".\"]}\n",
        "    )\n",
@@ -15001,10 +15001,10 @@
    "execution_count": 36,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:41:38.268023Z",
-     "iopub.status.busy": "2024-09-05T17:41:38.267717Z",
-     "iopub.status.idle": "2024-09-05T17:41:39.039142Z",
-     "shell.execute_reply": "2024-09-05T17:41:39.038561Z"
+     "iopub.execute_input": "2024-09-10T17:15:17.236275Z",
+     "iopub.status.busy": "2024-09-10T17:15:17.236104Z",
+     "iopub.status.idle": "2024-09-10T17:15:18.020746Z",
+     "shell.execute_reply": "2024-09-10T17:15:18.020221Z"
     }
    },
    "outputs": [
@@ -15102,10 +15102,10 @@
    "execution_count": 37,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:41:39.041285Z",
-     "iopub.status.busy": "2024-09-05T17:41:39.041107Z",
-     "iopub.status.idle": "2024-09-05T17:41:39.124224Z",
-     "shell.execute_reply": "2024-09-05T17:41:39.123625Z"
+     "iopub.execute_input": "2024-09-10T17:15:18.023154Z",
+     "iopub.status.busy": "2024-09-10T17:15:18.022722Z",
+     "iopub.status.idle": "2024-09-10T17:15:18.107023Z",
+     "shell.execute_reply": "2024-09-10T17:15:18.106462Z"
     }
    },
    "outputs": [
@@ -16191,9 +16191,9 @@
        }
       },
       "text/html": [
-       "<div>                            <div id=\"99df8c3d-54d9-4586-a67f-082a21e055a1\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"99df8c3d-54d9-4586-a67f-082a21e055a1\")) {                    Plotly.newPlot(                        \"99df8c3d-54d9-4586-a67f-082a21e055a1\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.03906984627246857,0.0004489120910875499,0.03133600950241089,0.007519586011767387,0.034592170268297195,0.0003623008378781378,0.03451276570558548,0.19740214943885803,0.03844786807894707,0.04053793475031853,0.027628779411315918,0.02496313489973545],[0.18906497955322266,0.1721990704536438,0.06807752698659897,0.044945165514945984,0.0790855661034584,0.03096737712621689,0.028282105922698975,0.036443259567022324,0.026936713606119156,0.018826233223080635,0.04510088637471199,0.0065726605243980885],[0.15745531022548676,0.020724497735500336,0.48180025815963745,0.2991351783275604,0.1076490506529808,0.33004093170166016,0.09975510090589523,0.04926132783293724,0.2549382448196411,0.3606453239917755,0.12571798264980316,0.07931824773550034],[0.0058439951390028,0.1578730344772339,0.4189079701900482,0.30129143595695496,0.014345025643706322,0.03234415501356125,0.3312898576259613,0.5285982489585876,0.34241825342178345,0.10183659940958023,0.10516025871038437,0.22331231832504272],[0.10626537352800369,0.11930901557207108,0.02288069576025009,0.22826972603797913,0.02000402845442295,0.10010094195604324,0.17392224073410034,0.17407013475894928,0.025876855477690697,0.10249904543161392,0.009514803998172283,0.9921424388885498],[0.019766751676797867,0.005283255595713854,0.166484072804451,0.12087711691856384,0.1649997979402542,0.008032665587961674,0.4176996946334839,0.02582792565226555,0.04802580177783966,0.016231726855039597,0.031101685017347336,0.024261072278022766],[0.21729078888893127,0.03909975662827492,0.018048642203211784,0.059900619089603424,0.0329345278441906,0.0873454138636589,0.026895388960838318,0.09439495950937271,0.49926066398620605,0.00624012341722846,0.027026763185858727,0.1278562843799591],[0.25116443634033203,0.013308697380125523,0.006663368083536625,0.03743036091327667,0.02331540174782276,0.017407197505235672,0.02206745184957981,0.022141238674521446,0.04502354562282562,0.02084278129041195,0.008310752920806408,0.017167769372463226],[0.020890571177005768,0.016537854447960854,0.021583056077361107,0.015005743131041527,0.024211907759308815,0.1019899770617485,0.029100272804498672,0.22793740034103394,0.02781473658978939,0.01794101856648922,0.02482905425131321,0.03806224465370178],[0.02607565000653267,0.015407402068376541,0.020444145426154137,0.14558184146881104,0.01247023232281208,0.017151616513729095,0.013311783783137798,0.02445186860859394,0.018111703917384148,0.013193286024034023,0.03573964536190033,0.0187982190400362],[0.02147809788584709,0.018419133499264717,0.0181836299598217,0.021721404045820236,0.03156769275665283,0.03470622003078461,0.017550060525536537,0.011417530477046967,0.015795614570379257,0.04592300206422806,0.016215480864048004,0.030394626781344414],[0.033205095678567886,0.017571475356817245,0.015131049789488316,0.04148414358496666,0.015181177295744419,0.01758996769785881,0.01514849066734314,0.017676066607236862,0.06622707098722458,0.01845112442970276,0.01700744405388832,0.02974928542971611]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Previous Token Scores\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "<div>                            <div id=\"a1e3ae9c-4880-482e-96c3-bfe0b0a07aa0\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"a1e3ae9c-4880-482e-96c3-bfe0b0a07aa0\")) {                    Plotly.newPlot(                        \"a1e3ae9c-4880-482e-96c3-bfe0b0a07aa0\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.03906984627246857,0.0004489120910875499,0.03133600950241089,0.007519586011767387,0.034592170268297195,0.0003623008378781378,0.03451276570558548,0.19740214943885803,0.03844786807894707,0.04053793475031853,0.027628779411315918,0.02496313489973545],[0.18906497955322266,0.1721990704536438,0.06807752698659897,0.044945165514945984,0.0790855661034584,0.03096737712621689,0.028282105922698975,0.036443259567022324,0.026936713606119156,0.018826233223080635,0.04510088637471199,0.0065726605243980885],[0.15745531022548676,0.020724497735500336,0.48180025815963745,0.2991351783275604,0.1076490506529808,0.33004093170166016,0.09975510090589523,0.04926132783293724,0.2549382448196411,0.3606453239917755,0.12571798264980316,0.07931824773550034],[0.0058439951390028,0.1578730344772339,0.4189079701900482,0.30129143595695496,0.014345025643706322,0.03234415501356125,0.3312898576259613,0.5285982489585876,0.34241825342178345,0.10183659940958023,0.10516025871038437,0.22331231832504272],[0.10626537352800369,0.11930901557207108,0.02288069576025009,0.22826972603797913,0.02000402845442295,0.10010094195604324,0.17392224073410034,0.17407013475894928,0.025876855477690697,0.10249904543161392,0.009514803998172283,0.9921424388885498],[0.019766751676797867,0.005283255595713854,0.166484072804451,0.12087711691856384,0.1649997979402542,0.008032665587961674,0.4176996946334839,0.02582792565226555,0.04802580177783966,0.016231726855039597,0.031101685017347336,0.024261072278022766],[0.21729078888893127,0.03909975662827492,0.018048642203211784,0.059900619089603424,0.0329345278441906,0.0873454138636589,0.026895388960838318,0.09439495950937271,0.49926066398620605,0.00624012341722846,0.027026763185858727,0.1278562843799591],[0.25116443634033203,0.013308697380125523,0.006663368083536625,0.03743036091327667,0.02331540174782276,0.017407197505235672,0.02206745184957981,0.022141238674521446,0.04502354562282562,0.02084278129041195,0.008310752920806408,0.017167769372463226],[0.020890571177005768,0.016537854447960854,0.021583056077361107,0.015005743131041527,0.024211907759308815,0.1019899770617485,0.029100272804498672,0.22793740034103394,0.02781473658978939,0.01794101856648922,0.02482905425131321,0.03806224465370178],[0.02607565000653267,0.015407402068376541,0.020444145426154137,0.14558184146881104,0.01247023232281208,0.017151616513729095,0.013311783783137798,0.02445186860859394,0.018111703917384148,0.013193286024034023,0.03573964536190033,0.0187982190400362],[0.02147809788584709,0.018419133499264717,0.0181836299598217,0.021721404045820236,0.03156769275665283,0.03470622003078461,0.017550060525536537,0.011417530477046967,0.015795614570379257,0.04592300206422806,0.016215480864048004,0.030394626781344414],[0.033205095678567886,0.017571475356817245,0.015131049789488316,0.04148414358496666,0.015181177295744419,0.01758996769785881,0.01514849066734314,0.017676066607236862,0.06622707098722458,0.01845112442970276,0.01700744405388832,0.02974928542971611]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Previous Token Scores\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('99df8c3d-54d9-4586-a67f-082a21e055a1');\n",
+       "var gd = document.getElementById('a1e3ae9c-4880-482e-96c3-bfe0b0a07aa0');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -17303,9 +17303,9 @@
        }
       },
       "text/html": [
-       "<div>                            <div id=\"87727d60-5c53-428f-b583-3c0bee78d87a\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"87727d60-5c53-428f-b583-3c0bee78d87a\")) {                    Plotly.newPlot(                        \"87727d60-5c53-428f-b583-3c0bee78d87a\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.003192325122654438,0.1323632299900055,0.005006915889680386,1.0427492270537186e-05,0.0013110269792377949,0.7034571170806885,0.004262051545083523,0.0001649669575272128,0.0024746304843574762,0.0008572910446673632,0.018891464918851852,0.008690940216183662],[0.0002916363882832229,0.00013781932648271322,0.0015036164550110698,0.005392488092184067,0.001858389819972217,0.009062976576387882,0.012414469383656979,0.00224055303260684,0.005135663319379091,0.005220625549554825,0.005546468310058117,0.029750702902674675],[0.0024816489312797785,0.009442185051739216,0.00034561342909000814,0.00025915156584233046,0.005211671814322472,0.0005709662800654769,0.0015209924895316362,0.006313098594546318,0.001560926903039217,0.00042158187716268003,0.0001535943301860243,0.005160389002412558],[0.6775667071342468,0.002840489149093628,0.0007841649930924177,0.004712563939392567,0.006322908215224743,0.00620671221986413,0.0005474375793710351,0.0003782894345931709,0.0020156530663371086,0.007952774874866009,0.003576836548745632,0.0026087891310453415],[0.008604029193520546,0.0070285736583173275,0.00759880430996418,0.0034427959471940994,0.016561321914196014,0.005979627370834351,0.0048697832971811295,0.0007624494028277695,0.006062139756977558,0.007536678109318018,0.012022883631289005,1.055450974014882e-12],[0.00950299110263586,0.008562120608985424,0.0041626025922596455,0.003008681582286954,0.006847443990409374,0.004358135629445314,0.007669272366911173,0.009584203362464905,0.007618824020028114,0.004328039940446615,0.04140293970704079,0.009761848486959934],[0.00445613032206893,0.008873283863067627,0.007405218668282032,0.006224926561117172,0.007319129537791014,0.005623883102089167,0.01734965480864048,0.005529423244297504,0.0029201731085777283,0.008636709302663803,0.006222232710570097,0.008358956314623356],[0.003699968568980694,0.041079357266426086,0.04148266464471817,0.009313643909990788,0.009097038768231869,0.008774377405643463,0.007298501208424568,0.023312222212553024,0.00884333811700344,0.00987985823303461,0.017598574981093407,0.006039811763912439],[0.008986336179077625,0.028667127713561058,0.00889119878411293,0.010114572942256927,0.009737403132021427,0.00761164166033268,0.009763195179402828,0.005155565217137337,0.009276354685425758,0.011895835399627686,0.010411957278847694,0.007498918566852808],[0.024409933015704155,0.011438388377428055,0.02003093995153904,0.005118591710925102,0.015081214718520641,0.012334463186562061,0.015452546067535877,0.008602438494563103,0.014702466316521168,0.020766181871294975,0.009192772209644318,0.005703327711671591],[0.017897022888064384,0.013280634768307209,0.0067551820538938046,0.012744802981615067,0.008020908571779728,0.007722198497503996,0.01734135113656521,0.0074547044932842255,0.007832478731870651,0.008252141997218132,0.013642732053995132,0.012807462364435196],[0.004923720378428698,0.007951012812554836,0.00794787798076868,0.004564068745821714,0.010363367386162281,0.009582011960446835,0.01028773095458746,0.008320694789290428,0.002570011653006077,0.012810957618057728,0.008063891902565956,0.0065582930110394955]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Duplicate Token Scores\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "<div>                            <div id=\"d0ab7dd5-0224-4a12-bfb5-77c7a94ad500\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"d0ab7dd5-0224-4a12-bfb5-77c7a94ad500\")) {                    Plotly.newPlot(                        \"d0ab7dd5-0224-4a12-bfb5-77c7a94ad500\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.003192325122654438,0.1323632299900055,0.005006915889680386,1.0427492270537186e-05,0.0013110269792377949,0.7034571170806885,0.004262051545083523,0.0001649669575272128,0.0024746304843574762,0.0008572910446673632,0.018891464918851852,0.008690940216183662],[0.0002916363882832229,0.00013781932648271322,0.0015036164550110698,0.005392488092184067,0.001858389819972217,0.009062976576387882,0.012414469383656979,0.00224055303260684,0.005135663319379091,0.005220625549554825,0.005546468310058117,0.029750702902674675],[0.0024816489312797785,0.009442185051739216,0.00034561342909000814,0.00025915156584233046,0.005211671814322472,0.0005709662800654769,0.0015209924895316362,0.006313098594546318,0.001560926903039217,0.00042158187716268003,0.0001535943301860243,0.005160389002412558],[0.6775667071342468,0.002840489149093628,0.0007841649930924177,0.004712563939392567,0.006322908215224743,0.00620671221986413,0.0005474375793710351,0.0003782894345931709,0.0020156530663371086,0.007952774874866009,0.003576836548745632,0.0026087891310453415],[0.008604029193520546,0.0070285736583173275,0.00759880430996418,0.0034427959471940994,0.016561321914196014,0.005979627370834351,0.0048697832971811295,0.0007624494028277695,0.006062139756977558,0.007536678109318018,0.012022883631289005,1.055450974014882e-12],[0.00950299110263586,0.008562120608985424,0.0041626025922596455,0.003008681582286954,0.006847443990409374,0.004358135629445314,0.007669272366911173,0.009584203362464905,0.007618824020028114,0.004328039940446615,0.04140293970704079,0.009761848486959934],[0.00445613032206893,0.008873283863067627,0.007405218668282032,0.006224926561117172,0.007319129537791014,0.005623883102089167,0.01734965480864048,0.005529423244297504,0.0029201731085777283,0.008636709302663803,0.006222232710570097,0.008358956314623356],[0.003699968568980694,0.041079357266426086,0.04148266464471817,0.009313643909990788,0.009097038768231869,0.008774377405643463,0.007298501208424568,0.023312222212553024,0.00884333811700344,0.00987985823303461,0.017598574981093407,0.006039811763912439],[0.008986336179077625,0.028667127713561058,0.00889119878411293,0.010114572942256927,0.009737403132021427,0.00761164166033268,0.009763195179402828,0.005155565217137337,0.009276354685425758,0.011895835399627686,0.010411957278847694,0.007498918566852808],[0.024409933015704155,0.011438388377428055,0.02003093995153904,0.005118591710925102,0.015081214718520641,0.012334463186562061,0.015452546067535877,0.008602438494563103,0.014702466316521168,0.020766181871294975,0.009192772209644318,0.005703327711671591],[0.017897022888064384,0.013280634768307209,0.0067551820538938046,0.012744802981615067,0.008020908571779728,0.007722198497503996,0.01734135113656521,0.0074547044932842255,0.007832478731870651,0.008252141997218132,0.013642732053995132,0.012807462364435196],[0.004923720378428698,0.007951012812554836,0.00794787798076868,0.004564068745821714,0.010363367386162281,0.009582011960446835,0.01028773095458746,0.008320694789290428,0.002570011653006077,0.012810957618057728,0.008063891902565956,0.0065582930110394955]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Duplicate Token Scores\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('87727d60-5c53-428f-b583-3c0bee78d87a');\n",
+       "var gd = document.getElementById('d0ab7dd5-0224-4a12-bfb5-77c7a94ad500');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -18415,9 +18415,9 @@
        }
       },
       "text/html": [
-       "<div>                            <div id=\"c40872d7-beb3-47ab-b81d-dccc77f7f064\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"c40872d7-beb3-47ab-b81d-dccc77f7f064\")) {                    Plotly.newPlot(                        \"c40872d7-beb3-47ab-b81d-dccc77f7f064\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.004035576246678829,3.85936327802483e-05,0.003946060314774513,1.7428237697458826e-07,5.9896454331465065e-05,4.0836926928022876e-05,0.003501761006191373,0.00024610935361124575,0.003167978022247553,0.003010402200743556,0.0020935393404215574,0.008525436744093895],[0.0005264790961518884,0.0001567041763337329,0.0015079419827088714,0.005595334805548191,0.0018401179695501924,0.003887568600475788,0.005349150858819485,0.0046491725370287895,0.005880181211978197,0.007283915765583515,0.005552181042730808,0.00012677432096097618],[0.0022015569265931845,0.008784865029156208,0.0021590511314570904,0.0010448142420500517,0.005142332520335913,0.002251654164865613,0.0008376692421734333,0.0063524022698402405,0.00261817779392004,0.0010309149511158466,0.00015219332999549806,0.005351166240870953],[0.007752286270260811,0.003091607242822647,0.00136294006370008,0.004341903142631054,0.011233086697757244,0.006535585504025221,0.0009068045765161514,0.0006078635924495757,0.0028195830527693033,0.005254121031612158,0.004195701330900192,0.0025541584473103285],[0.007342719938606024,0.004788315389305353,0.007458813022822142,0.0033073171507567167,0.007871860638260841,0.004219716414809227,0.004172018263489008,0.0005154716782271862,0.008124986663460732,0.006826961878687143,0.008085506968200207,3.761463363005646e-11],[0.43377670645713806,0.9306100010871887,0.006382290739566088,0.003473056945949793,0.005501003935933113,0.9255975484848022,0.005381471943110228,0.007857300341129303,0.008637800812721252,0.015764445066452026,0.012188390828669071,0.008265750482678413],[0.0025073012802749872,0.008432051166892052,0.008623287081718445,0.007653359789401293,0.011058016680181026,0.005525414831936359,0.017205143347382545,0.004794336389750242,0.004097685217857361,0.9257786273956299,0.020375533029437065,0.006313955411314964],[0.005555589683353901,0.18942901492118835,0.8509916663169861,0.008273174054920673,0.008239682763814926,0.00864996574819088,0.028328388929367065,0.08996234834194183,0.0066174231469631195,0.009413909167051315,0.9037811756134033,0.03037119098007679],[0.0073545570485293865,0.3791305124759674,0.005602711346000433,0.025401653721928596,0.008504705503582954,0.00623104115948081,0.1189238503575325,0.0051146638579666615,0.013350976631045341,0.01576736569404602,0.025844166055321693,0.008429795503616333],[0.23989056050777435,0.14378714561462402,0.09330623596906662,0.005819808691740036,0.07744759321212769,0.01644795574247837,0.4442824423313141,0.011141379363834858,0.03618974611163139,0.47264590859413147,0.008039995096623898,0.030952973291277885],[0.36065515875816345,0.4820112884044647,0.02285100519657135,0.12641923129558563,0.04125585779547691,0.007237402722239494,0.287715345621109,0.3897298574447632,0.0300607867538929,0.006112930364906788,0.16554805636405945,0.2224510908126831],[0.007408502046018839,0.033737070858478546,0.020412690937519073,0.0027554105035960674,0.02518630214035511,0.07808848470449448,0.033082809299230576,0.046439945697784424,0.0032543179113417864,0.2744251787662506,0.3800223767757416,0.009483428671956062]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Induction Head Scores\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "<div>                            <div id=\"db9eb568-d240-4b5e-ada7-72a996ee0efc\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"db9eb568-d240-4b5e-ada7-72a996ee0efc\")) {                    Plotly.newPlot(                        \"db9eb568-d240-4b5e-ada7-72a996ee0efc\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.004035576246678829,3.85936327802483e-05,0.003946060314774513,1.7428237697458826e-07,5.9896454331465065e-05,4.0836926928022876e-05,0.003501761006191373,0.00024610935361124575,0.003167978022247553,0.003010402200743556,0.0020935393404215574,0.008525436744093895],[0.0005264790961518884,0.0001567041763337329,0.0015079419827088714,0.005595334805548191,0.0018401179695501924,0.003887568600475788,0.005349150858819485,0.0046491725370287895,0.005880181211978197,0.007283915765583515,0.005552181042730808,0.00012677432096097618],[0.0022015569265931845,0.008784865029156208,0.0021590511314570904,0.0010448142420500517,0.005142332520335913,0.002251654164865613,0.0008376692421734333,0.0063524022698402405,0.00261817779392004,0.0010309149511158466,0.00015219332999549806,0.005351166240870953],[0.007752286270260811,0.003091607242822647,0.00136294006370008,0.004341903142631054,0.011233086697757244,0.006535585504025221,0.0009068045765161514,0.0006078635924495757,0.0028195830527693033,0.005254121031612158,0.004195701330900192,0.0025541584473103285],[0.007342719938606024,0.004788315389305353,0.007458813022822142,0.0033073171507567167,0.007871860638260841,0.004219716414809227,0.004172018263489008,0.0005154716782271862,0.008124986663460732,0.006826961878687143,0.008085506968200207,3.761463363005646e-11],[0.43377670645713806,0.9306100010871887,0.006382290739566088,0.003473056945949793,0.005501003935933113,0.9255975484848022,0.005381471943110228,0.007857300341129303,0.008637800812721252,0.015764445066452026,0.012188390828669071,0.008265750482678413],[0.0025073012802749872,0.008432051166892052,0.008623287081718445,0.007653359789401293,0.011058016680181026,0.005525414831936359,0.017205143347382545,0.004794336389750242,0.004097685217857361,0.9257786273956299,0.020375533029437065,0.006313955411314964],[0.005555589683353901,0.18942901492118835,0.8509916663169861,0.008273174054920673,0.008239682763814926,0.00864996574819088,0.028328388929367065,0.08996234834194183,0.0066174231469631195,0.009413909167051315,0.9037811756134033,0.03037119098007679],[0.0073545570485293865,0.3791305124759674,0.005602711346000433,0.025401653721928596,0.008504705503582954,0.00623104115948081,0.1189238503575325,0.0051146638579666615,0.013350976631045341,0.01576736569404602,0.025844166055321693,0.008429795503616333],[0.23989056050777435,0.14378714561462402,0.09330623596906662,0.005819808691740036,0.07744759321212769,0.01644795574247837,0.4442824423313141,0.011141379363834858,0.03618974611163139,0.47264590859413147,0.008039995096623898,0.030952973291277885],[0.36065515875816345,0.4820112884044647,0.02285100519657135,0.12641923129558563,0.04125585779547691,0.007237402722239494,0.287715345621109,0.3897298574447632,0.0300607867538929,0.006112930364906788,0.16554805636405945,0.2224510908126831],[0.007408502046018839,0.033737070858478546,0.020412690937519073,0.0027554105035960674,0.02518630214035511,0.07808848470449448,0.033082809299230576,0.046439945697784424,0.0032543179113417864,0.2744251787662506,0.3800223767757416,0.009483428671956062]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Induction Head Scores\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('c40872d7-beb3-47ab-b81d-dccc77f7f064');\n",
+       "var gd = document.getElementById('db9eb568-d240-4b5e-ada7-72a996ee0efc');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -18509,10 +18509,10 @@
    "execution_count": 38,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:41:39.126690Z",
-     "iopub.status.busy": "2024-09-05T17:41:39.126239Z",
-     "iopub.status.idle": "2024-09-05T17:41:39.412484Z",
-     "shell.execute_reply": "2024-09-05T17:41:39.411935Z"
+     "iopub.execute_input": "2024-09-10T17:15:18.109404Z",
+     "iopub.status.busy": "2024-09-10T17:15:18.109223Z",
+     "iopub.status.idle": "2024-09-10T17:15:18.402374Z",
+     "shell.execute_reply": "2024-09-10T17:15:18.401798Z"
     }
    },
    "outputs": [
@@ -18577,10 +18577,10 @@
    "execution_count": 39,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:41:39.415052Z",
-     "iopub.status.busy": "2024-09-05T17:41:39.414664Z",
-     "iopub.status.idle": "2024-09-05T17:41:39.506105Z",
-     "shell.execute_reply": "2024-09-05T17:41:39.505497Z"
+     "iopub.execute_input": "2024-09-10T17:15:18.404811Z",
+     "iopub.status.busy": "2024-09-10T17:15:18.404468Z",
+     "iopub.status.idle": "2024-09-10T17:15:18.486875Z",
+     "shell.execute_reply": "2024-09-10T17:15:18.486278Z"
     }
    },
    "outputs": [
@@ -19673,9 +19673,9 @@
        }
       },
       "text/html": [
-       "<div>                            <div id=\"63e4aa76-ac0f-4935-b710-49e27e2d83fb\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"63e4aa76-ac0f-4935-b710-49e27e2d83fb\")) {                    Plotly.newPlot(                        \"63e4aa76-ac0f-4935-b710-49e27e2d83fb\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[-0.002156492555513978,-0.00046507816296070814,0.00024167407536879182,0.00028066313825547695,-0.00041629592305980623,-0.0004892397555522621,-0.0026209561619907618,-0.0029356726445257664,0.000425610167440027,0.0005418087239377201,0.0002375440817559138,-7.489288691431284e-05],[-0.0006585267838090658,0.00040605987305752933,-0.0009330196771770716,0.0008937990060076118,-0.0009785185102373362,-0.0005338399787433445,-0.0027988061774522066,-0.004214102402329445,0.0025785816833376884,0.002450671512633562,0.0005351606523618102,0.0012349870521575212],[0.0009405002347193658,-0.0011168736964464188,-0.0011542305583134294,-0.0015697323251515627,-0.0005699749453924596,0.0014514662325382233,0.0024399266112595797,0.0031583067029714584,0.0009236747864633799,-0.00357811083085835,-0.0010650111362338066,-0.000355880125425756],[-0.0005624890327453613,-1.1902564438059926e-05,0.0011530898045748472,0.0007360299350693822,0.0016493811272084713,0.0008800593786872923,-0.0006905524642206728,-0.0030319970101118088,0.0008080819388851523,0.00010364333866164088,-0.0005807871930301189,-0.0011067159939557314],[-0.0026376438327133656,0.00026929602609016,-0.001641781534999609,-0.0034070422407239676,0.001744971377775073,0.00046448659850284457,-0.000789930927567184,0.0018328832229599357,-0.0008632286335341632,-0.00039789103902876377,0.000787912926170975,-0.00012039250577799976],[0.0008688485249876976,0.0009473453392274678,-0.0022809950169175863,-0.0011803111992776394,0.00024082366144284606,-0.0004318496794439852,-0.0003728233277797699,-0.0007385354256257415,0.0008114463416859508,-0.00040441250894218683,-0.007074240129441023,0.003946419805288315],[-0.01491759717464447,-0.002280194777995348,0.0022679578978568316,-8.297746535390615e-05,-0.004981024190783501,0.0027670864947140217,0.00626616645604372,-0.0034861797466874123,-0.0013347704662010074,-0.0017918730154633522,-0.001223177881911397,0.000405364902690053],[-0.00024617952294647694,-0.0057901544496417046,-0.0004975934862159193,0.14218278229236603,-0.001496216980740428,-0.01900637149810791,0.0031333654187619686,-0.0018582025077193975,-0.011305071413516998,0.19225990772247314,-0.0011892484035342932,-0.0010282367002218962],[-0.0038004019297659397,-0.0008571128128096461,-0.013956686481833458,0.008289236575365067,0.004314992111176252,-0.009073692373931408,-0.08315175771713257,0.0034568109549582005,-0.018055014312267303,0.00217800703831017,0.29780468344688416,0.02409377135336399],[0.08904657512903214,-0.0007931506261229515,0.07247606664896011,0.015016509220004082,-0.021209245547652245,0.052054572850465775,1.441115140914917,0.04743725806474686,-0.0322909839451313,0.0,0.0019993034657090902,-0.008072325959801674],[0.8600812554359436,0.3260071277618408,0.16344299912452698,0.07133564352989197,-0.004448350518941879,0.0006813818472437561,0.36613476276397705,-0.710505485534668,-0.0020313779823482037,-0.032143473625183105,1.2294319868087769,0.0018451482756063342],[0.016877301037311554,-0.001730009913444519,-0.501085638999939,0.02749774605035782,-0.005966278724372387,-0.0049441163428127766,-0.08855222165584564,0.006622320972383022,0.04412432014942169,-0.027266837656497955,-1.1349124908447266,0.02287965640425682]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"margin\":{\"t\":60}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "<div>                            <div id=\"17e8a56b-4819-438b-b954-b94f0a90174a\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"17e8a56b-4819-438b-b954-b94f0a90174a\")) {                    Plotly.newPlot(                        \"17e8a56b-4819-438b-b954-b94f0a90174a\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[-0.002156492555513978,-0.00046507816296070814,0.00024167407536879182,0.00028066313825547695,-0.00041629592305980623,-0.0004892397555522621,-0.0026209561619907618,-0.0029356726445257664,0.000425610167440027,0.0005418087239377201,0.0002375440817559138,-7.489288691431284e-05],[-0.0006585267838090658,0.00040605987305752933,-0.0009330196771770716,0.0008937990060076118,-0.0009785185102373362,-0.0005338399787433445,-0.0027988061774522066,-0.004214102402329445,0.0025785816833376884,0.002450671512633562,0.0005351606523618102,0.0012349870521575212],[0.0009405002347193658,-0.0011168736964464188,-0.0011542305583134294,-0.0015697323251515627,-0.0005699749453924596,0.0014514662325382233,0.0024399266112595797,0.0031583067029714584,0.0009236747864633799,-0.00357811083085835,-0.0010650111362338066,-0.000355880125425756],[-0.0005624890327453613,-1.1902564438059926e-05,0.0011530898045748472,0.0007360299350693822,0.0016493811272084713,0.0008800593786872923,-0.0006905524642206728,-0.0030319970101118088,0.0008080819388851523,0.00010364333866164088,-0.0005807871930301189,-0.0011067159939557314],[-0.0026376438327133656,0.00026929602609016,-0.001641781534999609,-0.0034070422407239676,0.001744971377775073,0.00046448659850284457,-0.000789930927567184,0.0018328832229599357,-0.0008632286335341632,-0.00039789103902876377,0.000787912926170975,-0.00012039250577799976],[0.0008688485249876976,0.0009473453392274678,-0.0022809950169175863,-0.0011803111992776394,0.00024082366144284606,-0.0004318496794439852,-0.0003728233277797699,-0.0007385354256257415,0.0008114463416859508,-0.00040441250894218683,-0.007074240129441023,0.003946419805288315],[-0.01491759717464447,-0.002280194777995348,0.0022679578978568316,-8.297746535390615e-05,-0.004981024190783501,0.0027670864947140217,0.00626616645604372,-0.0034861797466874123,-0.0013347704662010074,-0.0017918730154633522,-0.001223177881911397,0.000405364902690053],[-0.00024617952294647694,-0.0057901544496417046,-0.0004975934862159193,0.14218278229236603,-0.001496216980740428,-0.01900637149810791,0.0031333654187619686,-0.0018582025077193975,-0.011305071413516998,0.19225990772247314,-0.0011892484035342932,-0.0010282367002218962],[-0.0038004019297659397,-0.0008571128128096461,-0.013956686481833458,0.008289236575365067,0.004314992111176252,-0.009073692373931408,-0.08315175771713257,0.0034568109549582005,-0.018055014312267303,0.00217800703831017,0.29780468344688416,0.02409377135336399],[0.08904657512903214,-0.0007931506261229515,0.07247606664896011,0.015016509220004082,-0.021209245547652245,0.052054572850465775,1.441115140914917,0.04743725806474686,-0.0322909839451313,0.0,0.0019993034657090902,-0.008072325959801674],[0.8600812554359436,0.3260071277618408,0.16344299912452698,0.07133564352989197,-0.004448350518941879,0.0006813818472437561,0.36613476276397705,-0.710505485534668,-0.0020313779823482037,-0.032143473625183105,1.2294319868087769,0.0018451482756063342],[0.016877301037311554,-0.001730009913444519,-0.501085638999939,0.02749774605035782,-0.005966278724372387,-0.0049441163428127766,-0.08855222165584564,0.006622320972383022,0.04412432014942169,-0.027266837656497955,-1.1349124908447266,0.02287965640425682]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"margin\":{\"t\":60}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('63e4aa76-ac0f-4935-b710-49e27e2d83fb');\n",
+       "var gd = document.getElementById('17e8a56b-4819-438b-b954-b94f0a90174a');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -21018,9 +21018,9 @@
        }
       },
       "text/html": [
-       "<div>                            <div id=\"aed2d9cd-0ab8-4309-b50e-32156504e9cc\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"aed2d9cd-0ab8-4309-b50e-32156504e9cc\")) {                    Plotly.newPlot(                        \"aed2d9cd-0ab8-4309-b50e-32156504e9cc\",                        [{\"hovertemplate\":\"\\u003cb\\u003e%{hovertext}\\u003c\\u002fb\\u003e\\u003cbr\\u003e\\u003cbr\\u003eAblated=%{x}\\u003cbr\\u003eOriginal=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"hovertext\":[\"L0H0\",\"L0H1\",\"L0H2\",\"L0H3\",\"L0H4\",\"L0H5\",\"L0H6\",\"L0H7\",\"L0H8\",\"L0H9\",\"L0H10\",\"L0H11\",\"L1H0\",\"L1H1\",\"L1H2\",\"L1H3\",\"L1H4\",\"L1H5\",\"L1H6\",\"L1H7\",\"L1H8\",\"L1H9\",\"L1H10\",\"L1H11\",\"L2H0\",\"L2H1\",\"L2H2\",\"L2H3\",\"L2H4\",\"L2H5\",\"L2H6\",\"L2H7\",\"L2H8\",\"L2H9\",\"L2H10\",\"L2H11\",\"L3H0\",\"L3H1\",\"L3H2\",\"L3H3\",\"L3H4\",\"L3H5\",\"L3H6\",\"L3H7\",\"L3H8\",\"L3H9\",\"L3H10\",\"L3H11\",\"L4H0\",\"L4H1\",\"L4H2\",\"L4H3\",\"L4H4\",\"L4H5\",\"L4H6\",\"L4H7\",\"L4H8\",\"L4H9\",\"L4H10\",\"L4H11\",\"L5H0\",\"L5H1\",\"L5H2\",\"L5H3\",\"L5H4\",\"L5H5\",\"L5H6\",\"L5H7\",\"L5H8\",\"L5H9\",\"L5H10\",\"L5H11\",\"L6H0\",\"L6H1\",\"L6H2\",\"L6H3\",\"L6H4\",\"L6H5\",\"L6H6\",\"L6H7\",\"L6H8\",\"L6H9\",\"L6H10\",\"L6H11\",\"L7H0\",\"L7H1\",\"L7H2\",\"L7H3\",\"L7H4\",\"L7H5\",\"L7H6\",\"L7H7\",\"L7H8\",\"L7H9\",\"L7H10\",\"L7H11\",\"L8H0\",\"L8H1\",\"L8H2\",\"L8H3\",\"L8H4\",\"L8H5\",\"L8H6\",\"L8H7\",\"L8H8\",\"L8H9\",\"L8H10\",\"L8H11\",\"L9H0\",\"L9H1\",\"L9H2\",\"L9H3\",\"L9H4\",\"L9H5\",\"L9H6\",\"L9H7\",\"L9H8\",\"L9H9\",\"L9H10\",\"L9H11\",\"L10H0\",\"L10H1\",\"L10H2\",\"L10H3\",\"L10H4\",\"L10H5\",\"L10H6\",\"L10H7\",\"L10H8\",\"L10H9\",\"L10H10\",\"L10H11\",\"L11H0\",\"L11H1\",\"L11H2\",\"L11H3\",\"L11H4\",\"L11H5\",\"L11H6\",\"L11H7\",\"L11H8\",\"L11H9\",\"L11H10\",\"L11H11\"],\"legendgroup\":\"\",\"marker\":{\"color\":\"#636efa\",\"symbol\":\"circle\"},\"mode\":\"markers\",\"name\":\"\",\"orientation\":\"v\",\"showlegend\":false,\"x\":[-0.002156492555513978,-0.00046507816296070814,0.00024167407536879182,0.00028066313825547695,-0.00041629592305980623,-0.0004892397555522621,-0.0026209561619907618,-0.0029356726445257664,0.000425610167440027,0.0005418087239377201,0.0002375440817559138,-7.489288691431284e-05,-0.0006585267838090658,0.00040605987305752933,-0.0009330196771770716,0.0008937990060076118,-0.0009785185102373362,-0.0005338399787433445,-0.0027988061774522066,-0.004214102402329445,0.0025785816833376884,0.002450671512633562,0.0005351606523618102,0.0012349870521575212,0.0009405002347193658,-0.0011168736964464188,-0.0011542305583134294,-0.0015697323251515627,-0.0005699749453924596,0.0014514662325382233,0.0024399266112595797,0.0031583067029714584,0.0009236747864633799,-0.00357811083085835,-0.0010650111362338066,-0.000355880125425756,-0.0005624890327453613,-1.1902564438059926e-05,0.0011530898045748472,0.0007360299350693822,0.0016493811272084713,0.0008800593786872923,-0.0006905524642206728,-0.0030319970101118088,0.0008080819388851523,0.00010364333866164088,-0.0005807871930301189,-0.0011067159939557314,-0.0026376438327133656,0.00026929602609016,-0.001641781534999609,-0.0034070422407239676,0.001744971377775073,0.00046448659850284457,-0.000789930927567184,0.0018328832229599357,-0.0008632286335341632,-0.00039789103902876377,0.000787912926170975,-0.00012039250577799976,0.0008688485249876976,0.0009473453392274678,-0.0022809950169175863,-0.0011803111992776394,0.00024082366144284606,-0.0004318496794439852,-0.0003728233277797699,-0.0007385354256257415,0.0008114463416859508,-0.00040441250894218683,-0.007074240129441023,0.003946419805288315,-0.01491759717464447,-0.002280194777995348,0.0022679578978568316,-8.297746535390615e-05,-0.004981024190783501,0.0027670864947140217,0.00626616645604372,-0.0034861797466874123,-0.0013347704662010074,-0.0017918730154633522,-0.001223177881911397,0.000405364902690053,-0.00024617952294647694,-0.0057901544496417046,-0.0004975934862159193,0.14218278229236603,-0.001496216980740428,-0.01900637149810791,0.0031333654187619686,-0.0018582025077193975,-0.011305071413516998,0.19225990772247314,-0.0011892484035342932,-0.0010282367002218962,-0.0038004019297659397,-0.0008571128128096461,-0.013956686481833458,0.008289236575365067,0.004314992111176252,-0.009073692373931408,-0.08315175771713257,0.0034568109549582005,-0.018055014312267303,0.00217800703831017,0.29780468344688416,0.02409377135336399,0.08904657512903214,-0.0007931506261229515,0.07247606664896011,0.015016509220004082,-0.021209245547652245,0.052054572850465775,1.441115140914917,0.04743725806474686,-0.0322909839451313,0.0,0.0019993034657090902,-0.008072325959801674,0.8600812554359436,0.3260071277618408,0.16344299912452698,0.07133564352989197,-0.004448350518941879,0.0006813818472437561,0.36613476276397705,-0.710505485534668,-0.0020313779823482037,-0.032143473625183105,1.2294319868087769,0.0018451482756063342,0.016877301037311554,-0.001730009913444519,-0.501085638999939,0.02749774605035782,-0.005966278724372387,-0.0049441163428127766,-0.08855222165584564,0.006622320972383022,0.04412432014942169,-0.027266837656497955,-1.1349124908447266,0.02287965640425682],\"xaxis\":\"x\",\"y\":[-0.0020563285797834396,-0.0005101955030113459,0.0004685768508352339,0.00012511832755990326,-0.0006028721109032631,-0.00024295502225868404,-0.002318894723430276,-0.002758359769359231,0.0005645868368446827,0.000969740329310298,-0.0002504501899238676,4.740082658827305e-06,-0.0010071131400763988,0.0003947088844142854,-0.0015487205237150192,0.0014034901978448033,-0.0012652688892558217,-0.0011358250631019473,-0.0028159404173493385,-0.0029645359609276056,0.0029190238565206528,0.0025743518490344286,0.00036237656604498625,0.0017548884497955441,0.0005569332861341536,-0.00112663593608886,-0.0017354178708046675,-0.0014514722861349583,-0.00028740704874508083,0.0017210595542564988,0.00266590085811913,0.0031146793626248837,0.0005667305667884648,-0.0036664949730038643,-0.0018847067840397358,7.027178071439266e-06,-0.000726439815480262,0.00011369686399120837,0.001430142787285149,0.000749052269384265,0.0020184761378914118,0.0007436758605763316,-0.0004617759259417653,-0.003905785735696554,0.001140733016654849,-4.027335671707988e-05,-0.0013293256051838398,-0.0017636881675571203,-0.002828173339366913,0.0003364472358953208,-0.0014249038649722934,-0.0037773081567138433,0.0015998876187950373,0.00029889732832089067,-0.0008046309230849147,0.0020388164557516575,-0.0015593776479363441,-0.0006437147385440767,0.001116806990467012,-0.00035003889934159815,0.0011338151525706053,0.0011259106686338782,-0.0025163597892969847,-0.0014790240675210953,0.00038791983388364315,-6.407807813957334e-05,-0.0005096746608614922,-0.0008841876406222582,0.0006399309495463967,-0.001009696745313704,-0.0067590330727398396,0.0033667273819446564,-0.015147387981414795,-0.002135086804628372,0.002593189012259245,-0.00042674108408391476,-0.005559002980589867,0.0026659294962882996,0.006410874892026186,-0.0038270100485533476,-0.00038422830402851105,-0.0016430210089311004,-0.0013344308827072382,-9.184109512716532e-05,-9.488123760093004e-05,-0.005788922309875488,-0.0006383719155564904,0.134933739900589,-0.0017687628278508782,-0.018917974084615707,0.0038733629044145346,-0.002145076170563698,-0.010327237658202648,0.18325874209403992,-0.0007747883792035282,-0.0010452116839587688,-0.003833947703242302,-0.000804627372417599,-0.012673338875174522,0.008045812137424946,0.0036040153354406357,-0.009398169815540314,-0.08272106945514679,0.0035550352185964584,-0.018404126167297363,0.0017586719477549195,0.2896132469177246,0.022854045033454895,0.08595201373100281,-0.0006932567339390516,0.06816966831684113,0.01311141811311245,-0.021098004654049873,0.05112440511584282,1.384489893913269,0.04583733528852463,-0.038303446024656296,2.9854445457458496,0.001966139767318964,-0.008030213415622711,0.560872495174408,0.17083144187927246,-0.033618733286857605,0.05821547657251358,-0.0024530075024813414,0.0018771879840642214,0.2882729768753052,-1.898641586303711,-0.001528693363070488,-0.03513003885746002,0.48021769523620605,-0.0009116916917264462,0.0160758625715971,-0.03986112400889397,-0.3879111707210541,0.011123226955533028,-0.005477802362293005,-0.0025129495188593864,-0.08056114614009857,0.007518642581999302,0.043011054396629333,-0.0400824099779129,-0.9702335596084595,0.011862391605973244],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Ablated\"},\"range\":[-3,3]},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Original\"},\"range\":[-3,3]},\"legend\":{\"tracegroupgap\":0},\"title\":{\"text\":\"Original vs Post-Ablation Direct Logit Attribution of Heads\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "<div>                            <div id=\"6e7a7d6f-0a22-4385-834f-25d48524e13a\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"6e7a7d6f-0a22-4385-834f-25d48524e13a\")) {                    Plotly.newPlot(                        \"6e7a7d6f-0a22-4385-834f-25d48524e13a\",                        [{\"hovertemplate\":\"\\u003cb\\u003e%{hovertext}\\u003c\\u002fb\\u003e\\u003cbr\\u003e\\u003cbr\\u003eAblated=%{x}\\u003cbr\\u003eOriginal=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"hovertext\":[\"L0H0\",\"L0H1\",\"L0H2\",\"L0H3\",\"L0H4\",\"L0H5\",\"L0H6\",\"L0H7\",\"L0H8\",\"L0H9\",\"L0H10\",\"L0H11\",\"L1H0\",\"L1H1\",\"L1H2\",\"L1H3\",\"L1H4\",\"L1H5\",\"L1H6\",\"L1H7\",\"L1H8\",\"L1H9\",\"L1H10\",\"L1H11\",\"L2H0\",\"L2H1\",\"L2H2\",\"L2H3\",\"L2H4\",\"L2H5\",\"L2H6\",\"L2H7\",\"L2H8\",\"L2H9\",\"L2H10\",\"L2H11\",\"L3H0\",\"L3H1\",\"L3H2\",\"L3H3\",\"L3H4\",\"L3H5\",\"L3H6\",\"L3H7\",\"L3H8\",\"L3H9\",\"L3H10\",\"L3H11\",\"L4H0\",\"L4H1\",\"L4H2\",\"L4H3\",\"L4H4\",\"L4H5\",\"L4H6\",\"L4H7\",\"L4H8\",\"L4H9\",\"L4H10\",\"L4H11\",\"L5H0\",\"L5H1\",\"L5H2\",\"L5H3\",\"L5H4\",\"L5H5\",\"L5H6\",\"L5H7\",\"L5H8\",\"L5H9\",\"L5H10\",\"L5H11\",\"L6H0\",\"L6H1\",\"L6H2\",\"L6H3\",\"L6H4\",\"L6H5\",\"L6H6\",\"L6H7\",\"L6H8\",\"L6H9\",\"L6H10\",\"L6H11\",\"L7H0\",\"L7H1\",\"L7H2\",\"L7H3\",\"L7H4\",\"L7H5\",\"L7H6\",\"L7H7\",\"L7H8\",\"L7H9\",\"L7H10\",\"L7H11\",\"L8H0\",\"L8H1\",\"L8H2\",\"L8H3\",\"L8H4\",\"L8H5\",\"L8H6\",\"L8H7\",\"L8H8\",\"L8H9\",\"L8H10\",\"L8H11\",\"L9H0\",\"L9H1\",\"L9H2\",\"L9H3\",\"L9H4\",\"L9H5\",\"L9H6\",\"L9H7\",\"L9H8\",\"L9H9\",\"L9H10\",\"L9H11\",\"L10H0\",\"L10H1\",\"L10H2\",\"L10H3\",\"L10H4\",\"L10H5\",\"L10H6\",\"L10H7\",\"L10H8\",\"L10H9\",\"L10H10\",\"L10H11\",\"L11H0\",\"L11H1\",\"L11H2\",\"L11H3\",\"L11H4\",\"L11H5\",\"L11H6\",\"L11H7\",\"L11H8\",\"L11H9\",\"L11H10\",\"L11H11\"],\"legendgroup\":\"\",\"marker\":{\"color\":\"#636efa\",\"symbol\":\"circle\"},\"mode\":\"markers\",\"name\":\"\",\"orientation\":\"v\",\"showlegend\":false,\"x\":[-0.002156492555513978,-0.00046507816296070814,0.00024167407536879182,0.00028066313825547695,-0.00041629592305980623,-0.0004892397555522621,-0.0026209561619907618,-0.0029356726445257664,0.000425610167440027,0.0005418087239377201,0.0002375440817559138,-7.489288691431284e-05,-0.0006585267838090658,0.00040605987305752933,-0.0009330196771770716,0.0008937990060076118,-0.0009785185102373362,-0.0005338399787433445,-0.0027988061774522066,-0.004214102402329445,0.0025785816833376884,0.002450671512633562,0.0005351606523618102,0.0012349870521575212,0.0009405002347193658,-0.0011168736964464188,-0.0011542305583134294,-0.0015697323251515627,-0.0005699749453924596,0.0014514662325382233,0.0024399266112595797,0.0031583067029714584,0.0009236747864633799,-0.00357811083085835,-0.0010650111362338066,-0.000355880125425756,-0.0005624890327453613,-1.1902564438059926e-05,0.0011530898045748472,0.0007360299350693822,0.0016493811272084713,0.0008800593786872923,-0.0006905524642206728,-0.0030319970101118088,0.0008080819388851523,0.00010364333866164088,-0.0005807871930301189,-0.0011067159939557314,-0.0026376438327133656,0.00026929602609016,-0.001641781534999609,-0.0034070422407239676,0.001744971377775073,0.00046448659850284457,-0.000789930927567184,0.0018328832229599357,-0.0008632286335341632,-0.00039789103902876377,0.000787912926170975,-0.00012039250577799976,0.0008688485249876976,0.0009473453392274678,-0.0022809950169175863,-0.0011803111992776394,0.00024082366144284606,-0.0004318496794439852,-0.0003728233277797699,-0.0007385354256257415,0.0008114463416859508,-0.00040441250894218683,-0.007074240129441023,0.003946419805288315,-0.01491759717464447,-0.002280194777995348,0.0022679578978568316,-8.297746535390615e-05,-0.004981024190783501,0.0027670864947140217,0.00626616645604372,-0.0034861797466874123,-0.0013347704662010074,-0.0017918730154633522,-0.001223177881911397,0.000405364902690053,-0.00024617952294647694,-0.0057901544496417046,-0.0004975934862159193,0.14218278229236603,-0.001496216980740428,-0.01900637149810791,0.0031333654187619686,-0.0018582025077193975,-0.011305071413516998,0.19225990772247314,-0.0011892484035342932,-0.0010282367002218962,-0.0038004019297659397,-0.0008571128128096461,-0.013956686481833458,0.008289236575365067,0.004314992111176252,-0.009073692373931408,-0.08315175771713257,0.0034568109549582005,-0.018055014312267303,0.00217800703831017,0.29780468344688416,0.02409377135336399,0.08904657512903214,-0.0007931506261229515,0.07247606664896011,0.015016509220004082,-0.021209245547652245,0.052054572850465775,1.441115140914917,0.04743725806474686,-0.0322909839451313,0.0,0.0019993034657090902,-0.008072325959801674,0.8600812554359436,0.3260071277618408,0.16344299912452698,0.07133564352989197,-0.004448350518941879,0.0006813818472437561,0.36613476276397705,-0.710505485534668,-0.0020313779823482037,-0.032143473625183105,1.2294319868087769,0.0018451482756063342,0.016877301037311554,-0.001730009913444519,-0.501085638999939,0.02749774605035782,-0.005966278724372387,-0.0049441163428127766,-0.08855222165584564,0.006622320972383022,0.04412432014942169,-0.027266837656497955,-1.1349124908447266,0.02287965640425682],\"xaxis\":\"x\",\"y\":[-0.0020563285797834396,-0.0005101955030113459,0.0004685768508352339,0.00012511832755990326,-0.0006028721109032631,-0.00024295502225868404,-0.002318894723430276,-0.002758359769359231,0.0005645868368446827,0.000969740329310298,-0.0002504501899238676,4.740082658827305e-06,-0.0010071131400763988,0.0003947088844142854,-0.0015487205237150192,0.0014034901978448033,-0.0012652688892558217,-0.0011358250631019473,-0.0028159404173493385,-0.0029645359609276056,0.0029190238565206528,0.0025743518490344286,0.00036237656604498625,0.0017548884497955441,0.0005569332861341536,-0.00112663593608886,-0.0017354178708046675,-0.0014514722861349583,-0.00028740704874508083,0.0017210595542564988,0.00266590085811913,0.0031146793626248837,0.0005667305667884648,-0.0036664949730038643,-0.0018847067840397358,7.027178071439266e-06,-0.000726439815480262,0.00011369686399120837,0.001430142787285149,0.000749052269384265,0.0020184761378914118,0.0007436758605763316,-0.0004617759259417653,-0.003905785735696554,0.001140733016654849,-4.027335671707988e-05,-0.0013293256051838398,-0.0017636881675571203,-0.002828173339366913,0.0003364472358953208,-0.0014249038649722934,-0.0037773081567138433,0.0015998876187950373,0.00029889732832089067,-0.0008046309230849147,0.0020388164557516575,-0.0015593776479363441,-0.0006437147385440767,0.001116806990467012,-0.00035003889934159815,0.0011338151525706053,0.0011259106686338782,-0.0025163597892969847,-0.0014790240675210953,0.00038791983388364315,-6.407807813957334e-05,-0.0005096746608614922,-0.0008841876406222582,0.0006399309495463967,-0.001009696745313704,-0.0067590330727398396,0.0033667273819446564,-0.015147387981414795,-0.002135086804628372,0.002593189012259245,-0.00042674108408391476,-0.005559002980589867,0.0026659294962882996,0.006410874892026186,-0.0038270100485533476,-0.00038422830402851105,-0.0016430210089311004,-0.0013344308827072382,-9.184109512716532e-05,-9.488123760093004e-05,-0.005788922309875488,-0.0006383719155564904,0.134933739900589,-0.0017687628278508782,-0.018917974084615707,0.0038733629044145346,-0.002145076170563698,-0.010327237658202648,0.18325874209403992,-0.0007747883792035282,-0.0010452116839587688,-0.003833947703242302,-0.000804627372417599,-0.012673338875174522,0.008045812137424946,0.0036040153354406357,-0.009398169815540314,-0.08272106945514679,0.0035550352185964584,-0.018404126167297363,0.0017586719477549195,0.2896132469177246,0.022854045033454895,0.08595201373100281,-0.0006932567339390516,0.06816966831684113,0.01311141811311245,-0.021098004654049873,0.05112440511584282,1.384489893913269,0.04583733528852463,-0.038303446024656296,2.9854445457458496,0.001966139767318964,-0.008030213415622711,0.560872495174408,0.17083144187927246,-0.033618733286857605,0.05821547657251358,-0.0024530075024813414,0.0018771879840642214,0.2882729768753052,-1.898641586303711,-0.001528693363070488,-0.03513003885746002,0.48021769523620605,-0.0009116916917264462,0.0160758625715971,-0.03986112400889397,-0.3879111707210541,0.011123226955533028,-0.005477802362293005,-0.0025129495188593864,-0.08056114614009857,0.007518642581999302,0.043011054396629333,-0.0400824099779129,-0.9702335596084595,0.011862391605973244],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Ablated\"},\"range\":[-3,3]},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Original\"},\"range\":[-3,3]},\"legend\":{\"tracegroupgap\":0},\"title\":{\"text\":\"Original vs Post-Ablation Direct Logit Attribution of Heads\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('aed2d9cd-0ab8-4309-b50e-32156504e9cc');\n",
+       "var gd = document.getElementById('6e7a7d6f-0a22-4385-834f-25d48524e13a');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -21085,10 +21085,10 @@
    "execution_count": 40,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:41:39.509266Z",
-     "iopub.status.busy": "2024-09-05T17:41:39.509049Z",
-     "iopub.status.idle": "2024-09-05T17:41:39.514491Z",
-     "shell.execute_reply": "2024-09-05T17:41:39.513975Z"
+     "iopub.execute_input": "2024-09-10T17:15:18.489154Z",
+     "iopub.status.busy": "2024-09-10T17:15:18.488829Z",
+     "iopub.status.idle": "2024-09-10T17:15:18.494048Z",
+     "shell.execute_reply": "2024-09-10T17:15:18.493586Z"
     }
    },
    "outputs": [
@@ -21174,60 +21174,23 @@
   "widgets": {
    "application/vnd.jupyter.widget-state+json": {
     "state": {
-     "03f0891c87e343bcb0362f969ae5f46e": {
-      "model_module": "@jupyter-widgets/base",
+     "0251b82b322a4c019a7b949f50604da6": {
+      "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "LayoutModel",
+      "model_name": "ProgressStyleModel",
       "state": {
-       "_model_module": "@jupyter-widgets/base",
+       "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "LayoutModel",
+       "_model_name": "ProgressStyleModel",
        "_view_count": null,
        "_view_module": "@jupyter-widgets/base",
        "_view_module_version": "2.0.0",
-       "_view_name": "LayoutView",
-       "align_content": null,
-       "align_items": null,
-       "align_self": null,
-       "border_bottom": null,
-       "border_left": null,
-       "border_right": null,
-       "border_top": null,
-       "bottom": null,
-       "display": null,
-       "flex": null,
-       "flex_flow": null,
-       "grid_area": null,
-       "grid_auto_columns": null,
-       "grid_auto_flow": null,
-       "grid_auto_rows": null,
-       "grid_column": null,
-       "grid_gap": null,
-       "grid_row": null,
-       "grid_template_areas": null,
-       "grid_template_columns": null,
-       "grid_template_rows": null,
-       "height": null,
-       "justify_content": null,
-       "justify_items": null,
-       "left": null,
-       "margin": null,
-       "max_height": null,
-       "max_width": null,
-       "min_height": null,
-       "min_width": null,
-       "object_fit": null,
-       "object_position": null,
-       "order": null,
-       "overflow": null,
-       "padding": null,
-       "right": null,
-       "top": null,
-       "visibility": null,
-       "width": null
+       "_view_name": "StyleView",
+       "bar_color": null,
+       "description_width": ""
       }
      },
-     "094eda3d8c8e4ef4a71224ed2112888f": {
+     "0a4c123ed29c47bc87190f48ff5f90dd": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "ProgressStyleModel",
@@ -21243,7 +21206,7 @@
        "description_width": ""
       }
      },
-     "09c7749310034ebe804a345cddf59c63": {
+     "0fbe54bdec604d568897b4cd3c043dad": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HTMLStyleModel",
@@ -21261,7 +21224,7 @@
        "text_color": null
       }
      },
-     "1184e705dd634ff29325f8222cc7406e": {
+     "12c7e81fccbe41dda672b6b6722e8f98": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HBoxModel",
@@ -21276,16 +21239,39 @@
        "_view_name": "HBoxView",
        "box_style": "",
        "children": [
-        "IPY_MODEL_73d18ef08fc44b35878a0f0c98173961",
-        "IPY_MODEL_a59c6aa9b8ba42f5931d03545dd0b15a",
-        "IPY_MODEL_58e1228c5ada4acdb70c322dc097b209"
+        "IPY_MODEL_1d5596e7c1d84795868a2d32b89619ad",
+        "IPY_MODEL_83ff07e9d88341fba479467e7cadc460",
+        "IPY_MODEL_f5765110b6584f4bb805f71677f241b7"
        ],
-       "layout": "IPY_MODEL_03f0891c87e343bcb0362f969ae5f46e",
+       "layout": "IPY_MODEL_3901289b4d4a4337bb3bf0a7dcd8d5a4",
        "tabbable": null,
        "tooltip": null
       }
      },
-     "17380503ad0e41f98634db778d6f7928": {
+     "1d5596e7c1d84795868a2d32b89619ad": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "HTMLView",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_5b43729786974ebf96e850b126e3b2dd",
+       "placeholder": "​",
+       "style": "IPY_MODEL_cce8300a7d9247859a7461581998e4ce",
+       "tabbable": null,
+       "tooltip": null,
+       "value": "tokenizer_config.json: 100%"
+      }
+     },
+     "20480a53bb4e403fa078d8bf2fb4f0d1": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -21338,30 +21324,7 @@
        "width": null
       }
      },
-     "1fe6542d1e42421fab06b43a4911a479": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "HTMLView",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_20299bd8734d4ebe8b65a257c1124f15",
-       "placeholder": "​",
-       "style": "IPY_MODEL_49078358cf354e25b1d2c7f1b93c3d11",
-       "tabbable": null,
-       "tooltip": null,
-       "value": " 456k/456k [00:00&lt;00:00, 52.5MB/s]"
-      }
-     },
-     "20299bd8734d4ebe8b65a257c1124f15": {
+     "2216c5f5b6444ff38fff530386df077f": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -21414,30 +21377,31 @@
        "width": null
       }
      },
-     "21341249f6ef4c12ac7114db65da28db": {
+     "27abca058a2d442c9fa1488c14f5fe1c": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
+      "model_name": "HBoxModel",
       "state": {
        "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
+       "_model_name": "HBoxModel",
        "_view_count": null,
        "_view_module": "@jupyter-widgets/controls",
        "_view_module_version": "2.0.0",
-       "_view_name": "HTMLView",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_da5b7f87aab84ec1a34aa52a7ad90661",
-       "placeholder": "​",
-       "style": "IPY_MODEL_d60f3ddba73a4d51b5f9a5458225d767",
+       "_view_name": "HBoxView",
+       "box_style": "",
+       "children": [
+        "IPY_MODEL_be2d8de653174dc184d20219e9716cba",
+        "IPY_MODEL_f5c4159733c54649893abf1ab34201c1",
+        "IPY_MODEL_db72cb5fa9844c7b9087ff8d0e6e934c"
+       ],
+       "layout": "IPY_MODEL_2fd6e310c91a4be4af4e44f2ac0d65e4",
        "tabbable": null,
-       "tooltip": null,
-       "value": " 26.0/26.0 [00:00&lt;00:00, 4.56kB/s]"
+       "tooltip": null
       }
      },
-     "235b0d59d9f64192ae6e01f1ab05fb5d": {
+     "290303f0253140c1bd518e6e95f77b8d": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -21490,30 +21454,7 @@
        "width": null
       }
      },
-     "298f7209e9b24cb39e6d3b7c4079ff63": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "HTMLView",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_9751e1af80354cdd8adcd263569953b0",
-       "placeholder": "​",
-       "style": "IPY_MODEL_4c648589fd7b43ca80ac92f6ad748610",
-       "tabbable": null,
-       "tooltip": null,
-       "value": " 548M/548M [00:03&lt;00:00, 145MB/s]"
-      }
-     },
-     "2e73aff0bf6244ad95040c4082ee0d74": {
+     "2fd6e310c91a4be4af4e44f2ac0d65e4": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -21566,7 +21507,7 @@
        "width": null
       }
      },
-     "2f286d154df947d38a5ae5ccea2b8fad": {
+     "30b072ac28a94fa0a11b9803daddea18": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -21619,7 +21560,23 @@
        "width": null
       }
      },
-     "327522f058604bc4a47401f696ced8de": {
+     "30f82becd85f4ab6b8816b9312d1cdd5": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "ProgressStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "ProgressStyleModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "StyleView",
+       "bar_color": null,
+       "description_width": ""
+      }
+     },
+     "3105e32d09b24745b580cffafaa57174": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -21672,104 +21629,7 @@
        "width": null
       }
      },
-     "36e2c3b1cbd74c4697352bef36d3cd2b": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "HTMLView",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_ab8c1c1b39f948d29d4b0e0529342f27",
-       "placeholder": "​",
-       "style": "IPY_MODEL_c657cc795b0c46879505d7f6fa76f53e",
-       "tabbable": null,
-       "tooltip": null,
-       "value": " 124/124 [00:00&lt;00:00, 21.8kB/s]"
-      }
-     },
-     "38bfcf873a8a4d7fbf3002998c2b9e9e": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HBoxModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HBoxModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "HBoxView",
-       "box_style": "",
-       "children": [
-        "IPY_MODEL_68eaeabb3fba422a8d1bcd1cbb918576",
-        "IPY_MODEL_f45670d3a5504e4e8fb11ea1637ed193",
-        "IPY_MODEL_21341249f6ef4c12ac7114db65da28db"
-       ],
-       "layout": "IPY_MODEL_ef36e5539675442eba435bac9ff7cd7a",
-       "tabbable": null,
-       "tooltip": null
-      }
-     },
-     "3de70caac20d4c1586ed10bac3217294": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "ProgressStyleModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "ProgressStyleModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "bar_color": null,
-       "description_width": ""
-      }
-     },
-     "4617f97d43ab4c59a4887b23130d1902": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "ProgressStyleModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "ProgressStyleModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "bar_color": null,
-       "description_width": ""
-      }
-     },
-     "49078358cf354e25b1d2c7f1b93c3d11": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLStyleModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLStyleModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "background": null,
-       "description_width": "",
-       "font_size": null,
-       "text_color": null
-      }
-     },
-     "4c648589fd7b43ca80ac92f6ad748610": {
+     "340cbdd283674a878226f845ef97346b": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HTMLStyleModel",
@@ -21787,30 +21647,7 @@
        "text_color": null
       }
      },
-     "5151e930cb894560a9f9046725028fde": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "HTMLView",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_17380503ad0e41f98634db778d6f7928",
-       "placeholder": "​",
-       "style": "IPY_MODEL_09c7749310034ebe804a345cddf59c63",
-       "tabbable": null,
-       "tooltip": null,
-       "value": "model.safetensors: 100%"
-      }
-     },
-     "51a3da9a97644ba585cbdd682808c550": {
+     "386bba55285147a4aedf17cb5a7e0215": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -21863,30 +21700,60 @@
        "width": null
       }
      },
-     "58e1228c5ada4acdb70c322dc097b209": {
-      "model_module": "@jupyter-widgets/controls",
+     "3901289b4d4a4337bb3bf0a7dcd8d5a4": {
+      "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
+      "model_name": "LayoutModel",
       "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
+       "_model_module": "@jupyter-widgets/base",
        "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
+       "_model_name": "LayoutModel",
        "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
+       "_view_module": "@jupyter-widgets/base",
        "_view_module_version": "2.0.0",
-       "_view_name": "HTMLView",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_ad4c01375564485794658801d02bbb80",
-       "placeholder": "​",
-       "style": "IPY_MODEL_c57503aa752f4d889fc8fd1373ea4ad7",
-       "tabbable": null,
-       "tooltip": null,
-       "value": " 1.04M/1.04M [00:00&lt;00:00, 12.2MB/s]"
+       "_view_name": "LayoutView",
+       "align_content": null,
+       "align_items": null,
+       "align_self": null,
+       "border_bottom": null,
+       "border_left": null,
+       "border_right": null,
+       "border_top": null,
+       "bottom": null,
+       "display": null,
+       "flex": null,
+       "flex_flow": null,
+       "grid_area": null,
+       "grid_auto_columns": null,
+       "grid_auto_flow": null,
+       "grid_auto_rows": null,
+       "grid_column": null,
+       "grid_gap": null,
+       "grid_row": null,
+       "grid_template_areas": null,
+       "grid_template_columns": null,
+       "grid_template_rows": null,
+       "height": null,
+       "justify_content": null,
+       "justify_items": null,
+       "left": null,
+       "margin": null,
+       "max_height": null,
+       "max_width": null,
+       "min_height": null,
+       "min_width": null,
+       "object_fit": null,
+       "object_position": null,
+       "order": null,
+       "overflow": null,
+       "padding": null,
+       "right": null,
+       "top": null,
+       "visibility": null,
+       "width": null
       }
      },
-     "59dc2391b7964d3e9df04a6ba66f9ff1": {
+     "44a054ad7e3c4c59b2eeafe6008fa3dc": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HTMLStyleModel",
@@ -21904,7 +21771,7 @@
        "text_color": null
       }
      },
-     "6086e366713042c398600c11cb8615f2": {
+     "45eb78ab9b9c4813af6d66ce8726b8f4": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -21957,30 +21824,60 @@
        "width": null
       }
      },
-     "68eaeabb3fba422a8d1bcd1cbb918576": {
-      "model_module": "@jupyter-widgets/controls",
+     "46051a9cf3c747f8a5ec0234ff42bf09": {
+      "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
+      "model_name": "LayoutModel",
       "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
+       "_model_module": "@jupyter-widgets/base",
        "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
+       "_model_name": "LayoutModel",
        "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
+       "_view_module": "@jupyter-widgets/base",
        "_view_module_version": "2.0.0",
-       "_view_name": "HTMLView",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_327522f058604bc4a47401f696ced8de",
-       "placeholder": "​",
-       "style": "IPY_MODEL_8d1f546f867a4087a0467ce063107b44",
-       "tabbable": null,
-       "tooltip": null,
-       "value": "tokenizer_config.json: 100%"
+       "_view_name": "LayoutView",
+       "align_content": null,
+       "align_items": null,
+       "align_self": null,
+       "border_bottom": null,
+       "border_left": null,
+       "border_right": null,
+       "border_top": null,
+       "bottom": null,
+       "display": null,
+       "flex": null,
+       "flex_flow": null,
+       "grid_area": null,
+       "grid_auto_columns": null,
+       "grid_auto_flow": null,
+       "grid_auto_rows": null,
+       "grid_column": null,
+       "grid_gap": null,
+       "grid_row": null,
+       "grid_template_areas": null,
+       "grid_template_columns": null,
+       "grid_template_rows": null,
+       "height": null,
+       "justify_content": null,
+       "justify_items": null,
+       "left": null,
+       "margin": null,
+       "max_height": null,
+       "max_width": null,
+       "min_height": null,
+       "min_width": null,
+       "object_fit": null,
+       "object_position": null,
+       "order": null,
+       "overflow": null,
+       "padding": null,
+       "right": null,
+       "top": null,
+       "visibility": null,
+       "width": null
       }
      },
-     "71a8d19ff6ba415fb452440886c5359f": {
+     "48d38e46755540598972af4f8ffc5b83": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -22033,30 +21930,57 @@
        "width": null
       }
      },
-     "737dbee311b44b2f813fbc7d1e1e643d": {
+     "515362cd489440da86ee9da4b18234d4": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
+      "model_name": "HBoxModel",
       "state": {
        "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
+       "_model_name": "HBoxModel",
        "_view_count": null,
        "_view_module": "@jupyter-widgets/controls",
        "_view_module_version": "2.0.0",
-       "_view_name": "HTMLView",
+       "_view_name": "HBoxView",
+       "box_style": "",
+       "children": [
+        "IPY_MODEL_812086c57ba84bab86d979d1da3bc3a6",
+        "IPY_MODEL_707dfb3454114fc7bac50f699e0fa74b",
+        "IPY_MODEL_e683fa7576544b28af3a08e8c34fdb6d"
+       ],
+       "layout": "IPY_MODEL_290303f0253140c1bd518e6e95f77b8d",
+       "tabbable": null,
+       "tooltip": null
+      }
+     },
+     "58baa33b82404a59b690f3ad5d4a5bb1": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "FloatProgressModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "FloatProgressModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "ProgressView",
+       "bar_style": "success",
        "description": "",
        "description_allow_html": false,
-       "layout": "IPY_MODEL_235b0d59d9f64192ae6e01f1ab05fb5d",
-       "placeholder": "​",
-       "style": "IPY_MODEL_59dc2391b7964d3e9df04a6ba66f9ff1",
+       "layout": "IPY_MODEL_6b9a16110f364896abca2e2377be0a47",
+       "max": 124.0,
+       "min": 0.0,
+       "orientation": "horizontal",
+       "style": "IPY_MODEL_b3a416c3d9154ff7a1b2fb5ba1d1840c",
        "tabbable": null,
        "tooltip": null,
-       "value": "generation_config.json: 100%"
+       "value": 124.0
       }
      },
-     "73ac0538215847c6922f9a713c16f202": {
+     "5aa49a8508f84468b37311f1e3a7fa85": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HTMLStyleModel",
@@ -22074,30 +21998,7 @@
        "text_color": null
       }
      },
-     "73d18ef08fc44b35878a0f0c98173961": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "HTMLView",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_972adad22a2847ea88250929516670dd",
-       "placeholder": "​",
-       "style": "IPY_MODEL_80570890418a4de6b5e46e26ad3eb27d",
-       "tabbable": null,
-       "tooltip": null,
-       "value": "vocab.json: 100%"
-      }
-     },
-     "74157bf082f24cc7b4a199d44fac71b7": {
+     "5b43729786974ebf96e850b126e3b2dd": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -22150,7 +22051,31 @@
        "width": null
       }
      },
-     "75ec75df34b544548416c31c31de1186": {
+     "6325ae084ee34b078afc0e06f01c6afa": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HBoxModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HBoxModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "HBoxView",
+       "box_style": "",
+       "children": [
+        "IPY_MODEL_993ed5ffb8e443a8837d959250db0fab",
+        "IPY_MODEL_58baa33b82404a59b690f3ad5d4a5bb1",
+        "IPY_MODEL_6f5c1cf521994d24aa0ff115da16c22c"
+       ],
+       "layout": "IPY_MODEL_8fe8e25fa5794873b0b0b35a4745824d",
+       "tabbable": null,
+       "tooltip": null
+      }
+     },
+     "6b9a16110f364896abca2e2377be0a47": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -22203,7 +22128,7 @@
        "width": null
       }
      },
-     "75f59c523d3d4af7bf07923f5c5b8a8d": {
+     "6f5c1cf521994d24aa0ff115da16c22c": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HTMLModel",
@@ -22218,15 +22143,15 @@
        "_view_name": "HTMLView",
        "description": "",
        "description_allow_html": false,
-       "layout": "IPY_MODEL_51a3da9a97644ba585cbdd682808c550",
+       "layout": "IPY_MODEL_f608c20d677340f2b49f090bb72868c0",
        "placeholder": "​",
-       "style": "IPY_MODEL_cb669f7d109c4532aabbce076c46d5c0",
+       "style": "IPY_MODEL_340cbdd283674a878226f845ef97346b",
        "tabbable": null,
        "tooltip": null,
-       "value": " 1.36M/1.36M [00:00&lt;00:00, 7.22MB/s]"
+       "value": " 124/124 [00:00&lt;00:00, 21.4kB/s]"
       }
      },
-     "782b31eb7e404113aadff4df80d6d284": {
+     "707dfb3454114fc7bac50f699e0fa74b": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "FloatProgressModel",
@@ -22242,17 +22167,33 @@
        "bar_style": "success",
        "description": "",
        "description_allow_html": false,
-       "layout": "IPY_MODEL_6086e366713042c398600c11cb8615f2",
-       "max": 1355256.0,
+       "layout": "IPY_MODEL_f36154b87b0644debdd37df866ca67a1",
+       "max": 548105171.0,
        "min": 0.0,
        "orientation": "horizontal",
-       "style": "IPY_MODEL_adb3dd50e4e0478ebe832cbf0eda4c5d",
+       "style": "IPY_MODEL_93b110135ece4b46a8b80e31468b2ab2",
        "tabbable": null,
        "tooltip": null,
-       "value": 1355256.0
+       "value": 548105171.0
+      }
+     },
+     "7d80a0f33c4e48f0b0cef5bb8882aada": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "ProgressStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "ProgressStyleModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "StyleView",
+       "bar_color": null,
+       "description_width": ""
       }
      },
-     "7a06d497954c45c2a940b32fc5efa389": {
+     "80f8a6a0792d4a97ba6f083ae28b3218": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -22305,25 +22246,7 @@
        "width": null
       }
      },
-     "80570890418a4de6b5e46e26ad3eb27d": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLStyleModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLStyleModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "background": null,
-       "description_width": "",
-       "font_size": null,
-       "text_color": null
-      }
-     },
-     "83098f01fbf74522b5034d9a0458a055": {
+     "812086c57ba84bab86d979d1da3bc3a6": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HTMLModel",
@@ -22338,15 +22261,15 @@
        "_view_name": "HTMLView",
        "description": "",
        "description_allow_html": false,
-       "layout": "IPY_MODEL_e9cb28639f7f44bfb5f15db88b8b9f32",
+       "layout": "IPY_MODEL_db5d49ce14e24361b1be2eacc9479044",
        "placeholder": "​",
-       "style": "IPY_MODEL_73ac0538215847c6922f9a713c16f202",
+       "style": "IPY_MODEL_b5841c8cab9c4e97ba56ada551fdb2e0",
        "tabbable": null,
        "tooltip": null,
-       "value": "tokenizer.json: 100%"
+       "value": "model.safetensors: 100%"
       }
      },
-     "8476ed7c89264faa817152fd50d349a4": {
+     "839d3a22e0b546d7bc98f515c83ca90c": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -22399,7 +22322,56 @@
        "width": null
       }
      },
-     "8d1f546f867a4087a0467ce063107b44": {
+     "83ff07e9d88341fba479467e7cadc460": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "FloatProgressModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "FloatProgressModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "ProgressView",
+       "bar_style": "success",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_3105e32d09b24745b580cffafaa57174",
+       "max": 26.0,
+       "min": 0.0,
+       "orientation": "horizontal",
+       "style": "IPY_MODEL_7d80a0f33c4e48f0b0cef5bb8882aada",
+       "tabbable": null,
+       "tooltip": null,
+       "value": 26.0
+      }
+     },
+     "894f3b3042554c25aa4ceacd228af5c2": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "HTMLView",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_30b072ac28a94fa0a11b9803daddea18",
+       "placeholder": "​",
+       "style": "IPY_MODEL_a49e011ea5374814b360bde1cf84c71a",
+       "tabbable": null,
+       "tooltip": null,
+       "value": "tokenizer.json: 100%"
+      }
+     },
+     "8cbee9c01a5844769bc6ecac96eae6e9": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HTMLStyleModel",
@@ -22417,7 +22389,7 @@
        "text_color": null
       }
      },
-     "96488d53528247589f8268eac0dd6b05": {
+     "8fe8e25fa5794873b0b0b35a4745824d": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -22470,8 +22442,65 @@
        "width": null
       }
      },
-     "972adad22a2847ea88250929516670dd": {
-      "model_module": "@jupyter-widgets/base",
+     "921a2e40a82344bf99fbb499f83d755b": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "HTMLView",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_80f8a6a0792d4a97ba6f083ae28b3218",
+       "placeholder": "​",
+       "style": "IPY_MODEL_8cbee9c01a5844769bc6ecac96eae6e9",
+       "tabbable": null,
+       "tooltip": null,
+       "value": "vocab.json: 100%"
+      }
+     },
+     "93b110135ece4b46a8b80e31468b2ab2": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "ProgressStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "ProgressStyleModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "StyleView",
+       "bar_color": null,
+       "description_width": ""
+      }
+     },
+     "9780b9d58ebe4211a7a618aa44328af9": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLStyleModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "StyleView",
+       "background": null,
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
+      }
+     },
+     "9872396553d24753a6a9f1b37df96713": {
+      "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
       "state": {
@@ -22523,7 +22552,48 @@
        "width": null
       }
      },
-     "9751e1af80354cdd8adcd263569953b0": {
+     "993ed5ffb8e443a8837d959250db0fab": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "HTMLView",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_e4aa1ab25fbc49fa96419a7f093c3e4e",
+       "placeholder": "​",
+       "style": "IPY_MODEL_0fbe54bdec604d568897b4cd3c043dad",
+       "tabbable": null,
+       "tooltip": null,
+       "value": "generation_config.json: 100%"
+      }
+     },
+     "a49e011ea5374814b360bde1cf84c71a": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLStyleModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "StyleView",
+       "background": null,
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
+      }
+     },
+     "a96b44caa83f45df904765efdc61793b": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -22576,7 +22646,7 @@
        "width": null
       }
      },
-     "a02b083035864edd890d8cc04587840d": {
+     "aee08d1d53394415a2f7eb28925c5752": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "FloatProgressModel",
@@ -22592,67 +22662,110 @@
        "bar_style": "success",
        "description": "",
        "description_allow_html": false,
-       "layout": "IPY_MODEL_abaf254a67b84797a0e9e6680b17bf84",
-       "max": 456318.0,
+       "layout": "IPY_MODEL_48d38e46755540598972af4f8ffc5b83",
+       "max": 1042301.0,
        "min": 0.0,
        "orientation": "horizontal",
-       "style": "IPY_MODEL_f6fe61cf10ff4faca78694402dcc489a",
+       "style": "IPY_MODEL_0a4c123ed29c47bc87190f48ff5f90dd",
        "tabbable": null,
        "tooltip": null,
-       "value": 456318.0
+       "value": 1042301.0
       }
      },
-     "a05732c0ee01403fa202c360dc6bcc0b": {
+     "b3a416c3d9154ff7a1b2fb5ba1d1840c": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "FloatProgressModel",
+      "model_name": "ProgressStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "ProgressStyleModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "StyleView",
+       "bar_color": null,
+       "description_width": ""
+      }
+     },
+     "b5841c8cab9c4e97ba56ada551fdb2e0": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLStyleModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "StyleView",
+       "background": null,
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
+      }
+     },
+     "be2d8de653174dc184d20219e9716cba": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLModel",
       "state": {
        "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "FloatProgressModel",
+       "_model_name": "HTMLModel",
        "_view_count": null,
        "_view_module": "@jupyter-widgets/controls",
        "_view_module_version": "2.0.0",
-       "_view_name": "ProgressView",
-       "bar_style": "success",
+       "_view_name": "HTMLView",
        "description": "",
        "description_allow_html": false,
-       "layout": "IPY_MODEL_f8f6869de10d4fbeb8b380d43cad31d4",
-       "max": 548105171.0,
-       "min": 0.0,
-       "orientation": "horizontal",
-       "style": "IPY_MODEL_4617f97d43ab4c59a4887b23130d1902",
+       "layout": "IPY_MODEL_9872396553d24753a6a9f1b37df96713",
+       "placeholder": "​",
+       "style": "IPY_MODEL_c0896f84589a42d383d5b69e71979064",
        "tabbable": null,
        "tooltip": null,
-       "value": 548105171.0
+       "value": "merges.txt: 100%"
       }
      },
-     "a42e2bbeb4644be780fd859fae156453": {
+     "c0896f84589a42d383d5b69e71979064": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "HBoxModel",
+      "model_name": "HTMLStyleModel",
       "state": {
-       "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "HBoxModel",
+       "_model_name": "HTMLStyleModel",
        "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
+       "_view_module": "@jupyter-widgets/base",
        "_view_module_version": "2.0.0",
-       "_view_name": "HBoxView",
-       "box_style": "",
-       "children": [
-        "IPY_MODEL_5151e930cb894560a9f9046725028fde",
-        "IPY_MODEL_a05732c0ee01403fa202c360dc6bcc0b",
-        "IPY_MODEL_298f7209e9b24cb39e6d3b7c4079ff63"
-       ],
-       "layout": "IPY_MODEL_96488d53528247589f8268eac0dd6b05",
-       "tabbable": null,
-       "tooltip": null
+       "_view_name": "StyleView",
+       "background": null,
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
+      }
+     },
+     "cce8300a7d9247859a7461581998e4ce": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLStyleModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "StyleView",
+       "background": null,
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
       }
      },
-     "a59c6aa9b8ba42f5931d03545dd0b15a": {
+     "d0a71171b9454878a990f89c1e164277": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "FloatProgressModel",
@@ -22668,17 +22781,17 @@
        "bar_style": "success",
        "description": "",
        "description_allow_html": false,
-       "layout": "IPY_MODEL_2f286d154df947d38a5ae5ccea2b8fad",
-       "max": 1042301.0,
+       "layout": "IPY_MODEL_386bba55285147a4aedf17cb5a7e0215",
+       "max": 1355256.0,
        "min": 0.0,
        "orientation": "horizontal",
-       "style": "IPY_MODEL_3de70caac20d4c1586ed10bac3217294",
+       "style": "IPY_MODEL_30f82becd85f4ab6b8816b9312d1cdd5",
        "tabbable": null,
        "tooltip": null,
-       "value": 1042301.0
+       "value": 1355256.0
       }
      },
-     "ab8c1c1b39f948d29d4b0e0529342f27": {
+     "db5d49ce14e24361b1be2eacc9479044": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -22731,7 +22844,53 @@
        "width": null
       }
      },
-     "abaf254a67b84797a0e9e6680b17bf84": {
+     "db72cb5fa9844c7b9087ff8d0e6e934c": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "HTMLView",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_2216c5f5b6444ff38fff530386df077f",
+       "placeholder": "​",
+       "style": "IPY_MODEL_5aa49a8508f84468b37311f1e3a7fa85",
+       "tabbable": null,
+       "tooltip": null,
+       "value": " 456k/456k [00:00&lt;00:00, 65.2MB/s]"
+      }
+     },
+     "de96e35e533a420c960013bfb1ead429": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "HTMLView",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_45eb78ab9b9c4813af6d66ce8726b8f4",
+       "placeholder": "​",
+       "style": "IPY_MODEL_f9f8d47155be4081b7f9ed66c6350cd6",
+       "tabbable": null,
+       "tooltip": null,
+       "value": " 1.04M/1.04M [00:00&lt;00:00, 36.1MB/s]"
+      }
+     },
+     "e4aa1ab25fbc49fa96419a7f093c3e4e": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -22784,7 +22943,30 @@
        "width": null
       }
      },
-     "ad4c01375564485794658801d02bbb80": {
+     "e683fa7576544b28af3a08e8c34fdb6d": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "HTMLView",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_839d3a22e0b546d7bc98f515c83ca90c",
+       "placeholder": "​",
+       "style": "IPY_MODEL_f0f9e7c2bd7141449c36e529084c0b2b",
+       "tabbable": null,
+       "tooltip": null,
+       "value": " 548M/548M [00:01&lt;00:00, 287MB/s]"
+      }
+     },
+     "eebc36e492664b00b22a70b251d2e44d": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -22837,107 +23019,30 @@
        "width": null
       }
      },
-     "adb3dd50e4e0478ebe832cbf0eda4c5d": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "ProgressStyleModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "ProgressStyleModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "bar_color": null,
-       "description_width": ""
-      }
-     },
-     "b35dabf61035460aa1dd0200cf89fa93": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HBoxModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HBoxModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "HBoxView",
-       "box_style": "",
-       "children": [
-        "IPY_MODEL_83098f01fbf74522b5034d9a0458a055",
-        "IPY_MODEL_782b31eb7e404113aadff4df80d6d284",
-        "IPY_MODEL_75f59c523d3d4af7bf07923f5c5b8a8d"
-       ],
-       "layout": "IPY_MODEL_71a8d19ff6ba415fb452440886c5359f",
-       "tabbable": null,
-       "tooltip": null
-      }
-     },
-     "b9a1fa123e9f4bd89dad50aac41fcff5": {
+     "ef62f1509335401ea8b186ea892dcf60": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "HBoxModel",
+      "model_name": "HTMLModel",
       "state": {
        "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "HBoxModel",
+       "_model_name": "HTMLModel",
        "_view_count": null,
        "_view_module": "@jupyter-widgets/controls",
        "_view_module_version": "2.0.0",
-       "_view_name": "HBoxView",
-       "box_style": "",
-       "children": [
-        "IPY_MODEL_d4ce5dfcb1c64b039e68d2d08d7fa72c",
-        "IPY_MODEL_a02b083035864edd890d8cc04587840d",
-        "IPY_MODEL_1fe6542d1e42421fab06b43a4911a479"
-       ],
-       "layout": "IPY_MODEL_2e73aff0bf6244ad95040c4082ee0d74",
+       "_view_name": "HTMLView",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_46051a9cf3c747f8a5ec0234ff42bf09",
+       "placeholder": "​",
+       "style": "IPY_MODEL_9780b9d58ebe4211a7a618aa44328af9",
        "tabbable": null,
-       "tooltip": null
-      }
-     },
-     "c57503aa752f4d889fc8fd1373ea4ad7": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLStyleModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLStyleModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "background": null,
-       "description_width": "",
-       "font_size": null,
-       "text_color": null
-      }
-     },
-     "c657cc795b0c46879505d7f6fa76f53e": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLStyleModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLStyleModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "background": null,
-       "description_width": "",
-       "font_size": null,
-       "text_color": null
+       "tooltip": null,
+       "value": " 1.36M/1.36M [00:00&lt;00:00, 32.5MB/s]"
       }
      },
-     "cb669f7d109c4532aabbce076c46d5c0": {
+     "f0f9e7c2bd7141449c36e529084c0b2b": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HTMLStyleModel",
@@ -22955,48 +23060,31 @@
        "text_color": null
       }
      },
-     "d4ce5dfcb1c64b039e68d2d08d7fa72c": {
+     "f10863d1bbd14b91819db0b9074aa85a": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
+      "model_name": "HBoxModel",
       "state": {
        "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
+       "_model_name": "HBoxModel",
        "_view_count": null,
        "_view_module": "@jupyter-widgets/controls",
        "_view_module_version": "2.0.0",
-       "_view_name": "HTMLView",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_7a06d497954c45c2a940b32fc5efa389",
-       "placeholder": "​",
-       "style": "IPY_MODEL_ec4586211068491b9a0298a5323b0926",
+       "_view_name": "HBoxView",
+       "box_style": "",
+       "children": [
+        "IPY_MODEL_894f3b3042554c25aa4ceacd228af5c2",
+        "IPY_MODEL_d0a71171b9454878a990f89c1e164277",
+        "IPY_MODEL_ef62f1509335401ea8b186ea892dcf60"
+       ],
+       "layout": "IPY_MODEL_ffbeb3144f08405e8289aaed02f79c8a",
        "tabbable": null,
-       "tooltip": null,
-       "value": "merges.txt: 100%"
-      }
-     },
-     "d60f3ddba73a4d51b5f9a5458225d767": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLStyleModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLStyleModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "background": null,
-       "description_width": "",
-       "font_size": null,
-       "text_color": null
+       "tooltip": null
       }
      },
-     "da5b7f87aab84ec1a34aa52a7ad90661": {
+     "f36154b87b0644debdd37df866ca67a1": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -23049,104 +23137,80 @@
        "width": null
       }
      },
-     "e2cc4fea81124e6b988f5ebdb35b756f": {
+     "f4d8cb4603f64164bba7cbecd423516d": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "FloatProgressModel",
+      "model_name": "HBoxModel",
       "state": {
        "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "FloatProgressModel",
+       "_model_name": "HBoxModel",
        "_view_count": null,
        "_view_module": "@jupyter-widgets/controls",
        "_view_module_version": "2.0.0",
-       "_view_name": "ProgressView",
-       "bar_style": "success",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_8476ed7c89264faa817152fd50d349a4",
-       "max": 124.0,
-       "min": 0.0,
-       "orientation": "horizontal",
-       "style": "IPY_MODEL_f31f8342fa4b494fa56fe0efd5549cdd",
+       "_view_name": "HBoxView",
+       "box_style": "",
+       "children": [
+        "IPY_MODEL_921a2e40a82344bf99fbb499f83d755b",
+        "IPY_MODEL_aee08d1d53394415a2f7eb28925c5752",
+        "IPY_MODEL_de96e35e533a420c960013bfb1ead429"
+       ],
+       "layout": "IPY_MODEL_20480a53bb4e403fa078d8bf2fb4f0d1",
        "tabbable": null,
-       "tooltip": null,
-       "value": 124.0
+       "tooltip": null
       }
      },
-     "e9cb28639f7f44bfb5f15db88b8b9f32": {
-      "model_module": "@jupyter-widgets/base",
+     "f5765110b6584f4bb805f71677f241b7": {
+      "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "LayoutModel",
+      "model_name": "HTMLModel",
       "state": {
-       "_model_module": "@jupyter-widgets/base",
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "LayoutModel",
+       "_model_name": "HTMLModel",
        "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
+       "_view_module": "@jupyter-widgets/controls",
        "_view_module_version": "2.0.0",
-       "_view_name": "LayoutView",
-       "align_content": null,
-       "align_items": null,
-       "align_self": null,
-       "border_bottom": null,
-       "border_left": null,
-       "border_right": null,
-       "border_top": null,
-       "bottom": null,
-       "display": null,
-       "flex": null,
-       "flex_flow": null,
-       "grid_area": null,
-       "grid_auto_columns": null,
-       "grid_auto_flow": null,
-       "grid_auto_rows": null,
-       "grid_column": null,
-       "grid_gap": null,
-       "grid_row": null,
-       "grid_template_areas": null,
-       "grid_template_columns": null,
-       "grid_template_rows": null,
-       "height": null,
-       "justify_content": null,
-       "justify_items": null,
-       "left": null,
-       "margin": null,
-       "max_height": null,
-       "max_width": null,
-       "min_height": null,
-       "min_width": null,
-       "object_fit": null,
-       "object_position": null,
-       "order": null,
-       "overflow": null,
-       "padding": null,
-       "right": null,
-       "top": null,
-       "visibility": null,
-       "width": null
+       "_view_name": "HTMLView",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_a96b44caa83f45df904765efdc61793b",
+       "placeholder": "​",
+       "style": "IPY_MODEL_44a054ad7e3c4c59b2eeafe6008fa3dc",
+       "tabbable": null,
+       "tooltip": null,
+       "value": " 26.0/26.0 [00:00&lt;00:00, 4.39kB/s]"
       }
      },
-     "ec4586211068491b9a0298a5323b0926": {
+     "f5c4159733c54649893abf1ab34201c1": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "HTMLStyleModel",
+      "model_name": "FloatProgressModel",
       "state": {
+       "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "HTMLStyleModel",
+       "_model_name": "FloatProgressModel",
        "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
+       "_view_module": "@jupyter-widgets/controls",
        "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "background": null,
-       "description_width": "",
-       "font_size": null,
-       "text_color": null
+       "_view_name": "ProgressView",
+       "bar_style": "success",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_eebc36e492664b00b22a70b251d2e44d",
+       "max": 456318.0,
+       "min": 0.0,
+       "orientation": "horizontal",
+       "style": "IPY_MODEL_0251b82b322a4c019a7b949f50604da6",
+       "tabbable": null,
+       "tooltip": null,
+       "value": 456318.0
       }
      },
-     "ef36e5539675442eba435bac9ff7cd7a": {
+     "f608c20d677340f2b49f090bb72868c0": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -23199,65 +23263,25 @@
        "width": null
       }
      },
-     "f31f8342fa4b494fa56fe0efd5549cdd": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "ProgressStyleModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "ProgressStyleModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "bar_color": null,
-       "description_width": ""
-      }
-     },
-     "f45670d3a5504e4e8fb11ea1637ed193": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "FloatProgressModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "FloatProgressModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "ProgressView",
-       "bar_style": "success",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_74157bf082f24cc7b4a199d44fac71b7",
-       "max": 26.0,
-       "min": 0.0,
-       "orientation": "horizontal",
-       "style": "IPY_MODEL_094eda3d8c8e4ef4a71224ed2112888f",
-       "tabbable": null,
-       "tooltip": null,
-       "value": 26.0
-      }
-     },
-     "f6fe61cf10ff4faca78694402dcc489a": {
+     "f9f8d47155be4081b7f9ed66c6350cd6": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "ProgressStyleModel",
+      "model_name": "HTMLStyleModel",
       "state": {
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "ProgressStyleModel",
+       "_model_name": "HTMLStyleModel",
        "_view_count": null,
        "_view_module": "@jupyter-widgets/base",
        "_view_module_version": "2.0.0",
        "_view_name": "StyleView",
-       "bar_color": null,
-       "description_width": ""
+       "background": null,
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
       }
      },
-     "f8f6869de10d4fbeb8b380d43cad31d4": {
+     "ffbeb3144f08405e8289aaed02f79c8a": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -23309,30 +23333,6 @@
        "visibility": null,
        "width": null
       }
-     },
-     "f98e0318bcde48c7bc9ff7fa4c098c64": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HBoxModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HBoxModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "HBoxView",
-       "box_style": "",
-       "children": [
-        "IPY_MODEL_737dbee311b44b2f813fbc7d1e1e643d",
-        "IPY_MODEL_e2cc4fea81124e6b988f5ebdb35b756f",
-        "IPY_MODEL_36e2c3b1cbd74c4697352bef36d3cd2b"
-       ],
-       "layout": "IPY_MODEL_75ec75df34b544548416c31c31de1186",
-       "tabbable": null,
-       "tooltip": null
-      }
      }
     },
     "version_major": 2,
diff --git a/generated/demos/Main_Demo.html b/generated/demos/Main_Demo.html
index fd4554189..aae01df5e 100644
--- a/generated/demos/Main_Demo.html
+++ b/generated/demos/Main_Demo.html
@@ -385,11 +385,11 @@ <h1>Setup<a class="headerlink" href="#Setup" title="Permalink to this heading">#
 </pre></div>
 </div>
 <div class="output_area rendered_html docutils container">
-<div id="circuits-vis-2e3c0ab6-b350" style="margin: 15px 0;"/>
+<div id="circuits-vis-8b570228-4d3b" style="margin: 15px 0;"/>
     <script crossorigin type="module">
     import { render, Hello } from "https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js";
     render(
-      "circuits-vis-2e3c0ab6-b350",
+      "circuits-vis-8b570228-4d3b",
       Hello,
       {"name": "Neel"}
     )
@@ -440,7 +440,7 @@ <h1>Setup<a class="headerlink" href="#Setup" title="Permalink to this heading">#
 </div>
 <div class="output_area docutils container">
 <div class="highlight"><pre>
-&lt;torch.autograd.grad_mode.set_grad_enabled at 0x7f208dc6d3d0&gt;
+&lt;torch.autograd.grad_mode.set_grad_enabled at 0x7f282d327510&gt;
 </pre></div></div>
 </div>
 <p>Plotting helper functions:</p>
@@ -593,11 +593,11 @@ <h2>Caching all Activations<a class="headerlink" href="#Caching-all-Activations"
 </pre></div>
 </div>
 <div class="output_area rendered_html docutils container">
-<div id="circuits-vis-a7b9010c-0065" style="margin: 15px 0;"/>
+<div id="circuits-vis-7b1e6e9f-a1f2" style="margin: 15px 0;"/>
     <script crossorigin type="module">
     import { render, AttentionPatterns } from "https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js";
     render(
-      "circuits-vis-a7b9010c-0065",
+      "circuits-vis-7b1e6e9f-a1f2",
       AttentionPatterns,
       {"tokens": ["<|endoftext|>", "Natural", " language", " processing", " tasks", ",", " such", " as", " question", " answering", ",", " machine", " translation", ",", " reading", " comprehension", ",", " and", " summar", "ization", ",", " are", " typically", " approached", " with", " supervised", " learning", " on", " tasks", "pe", "cific", " datasets", "."], "attention": [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9639418125152588, 0.03605816140770912, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8389372825622559, 0.11828788369894028, 0.04277484491467476, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.47436124086380005, 0.13382026553153992, 0.27371731400489807, 0.11810113489627838, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.35606440901756287, 0.10184910148382187, 0.23054221272468567, 0.20397402346134186, 0.10757026076316833, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6660143733024597, 0.16866375505924225, 0.04535674676299095, 0.038855016231536865, 0.0677548199892044, 0.013355279341340065, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.38626962900161743, 0.285109281539917, 0.07609007507562637, 0.05908379331231117, 0.07223352044820786, 0.03979635238647461, 0.08141742646694183, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3775395154953003, 0.1883881539106369, 0.11723985522985458, 0.08685600012540817, 0.0666918158531189, 0.03500017523765564, 0.09693004935979843, 0.03135441616177559, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4869752824306488, 0.06781317293643951, 0.07952874153852463, 0.0848078578710556, 0.1590261608362198, 0.029577823355793953, 0.025685923174023628, 0.016474608331918716, 0.05011039599776268, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2906550168991089, 0.0401349738240242, 0.14614859223365784, 0.09940590709447861, 0.1538919061422348, 0.039001598954200745, 0.024988971650600433, 0.03184127062559128, 0.10222824662923813, 0.0717034563422203, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.39624103903770447, 0.09694176912307739, 0.027270659804344177, 0.02355135791003704, 0.03723449259996414, 0.006502415519207716, 0.08118756860494614, 0.013088458217680454, 0.06990590691566467, 0.24043099582195282, 0.007645336911082268, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.24864791333675385, 0.1380205899477005, 0.0923532024025917, 0.08676131069660187, 0.1381969302892685, 0.05914197862148285, 0.03223859518766403, 0.03158237785100937, 0.030489422380924225, 0.03873484209179878, 0.06671839207410812, 0.037114467471838, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1914844512939453, 0.1617259681224823, 0.07445938140153885, 0.07740949839353561, 0.021961113438010216, 0.03392130509018898, 0.05125021934509277, 0.01951923966407776, 0.03132446110248566, 0.04020152986049652, 0.038742680102586746, 0.21578852832317352, 0.042211681604385376, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3704318404197693, 0.08681417256593704, 0.02458467148244381, 0.021616315469145775, 0.032388731837272644, 0.005422737915068865, 0.0727522075176239, 0.0112727927044034, 0.06329693645238876, 0.21726809442043304, 0.006367155350744724, 0.029603807255625725, 0.05099846422672272, 0.007182050962001085, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1973765343427658, 0.04603995382785797, 0.0439998134970665, 0.1337345838546753, 0.05424821376800537, 0.025475721806287766, 0.027563506737351418, 0.021570932120084763, 0.051718227565288544, 0.06458097696304321, 0.02806464210152626, 0.23551592230796814, 0.019129814580082893, 0.029963519424200058, 0.021017681807279587, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08907236158847809, 0.01928834617137909, 0.1665353924036026, 0.07281264662742615, 0.04738640412688255, 0.024487905204296112, 0.028987322002649307, 0.019370367750525475, 0.026673022657632828, 0.07316635549068451, 0.025704585015773773, 0.04242359474301338, 0.058694612234830856, 0.028932694345712662, 0.18119072914123535, 0.09527372568845749, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2816729247570038, 0.06441289931535721, 0.018008552491664886, 0.01616962067782879, 0.023183872923254967, 0.0037532988935709, 0.05472247675061226, 0.007909760810434818, 0.046164702624082565, 0.16947267949581146, 0.004361647181212902, 0.021011337637901306, 0.0354907400906086, 0.004932564217597246, 0.0955522358417511, 0.14726325869560242, 0.005917454604059458, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.21305488049983978, 0.0591236874461174, 0.03382088243961334, 0.02747686579823494, 0.02839355543255806, 0.008422904647886753, 0.0400853231549263, 0.011629256419837475, 0.05295189097523689, 0.15404635667800903, 0.009831804782152176, 0.03610190004110336, 0.04737289249897003, 0.011069180443882942, 0.09972473233938217, 0.1397135704755783, 0.013185348361730576, 0.013994931243360043, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15871697664260864, 0.04387888312339783, 0.08712151646614075, 0.08998466283082962, 0.030738582834601402, 0.0341489352285862, 0.024917256087064743, 0.03139195218682289, 0.024823859333992004, 0.019790329039096832, 0.03625484183430672, 0.020694412291049957, 0.042840685695409775, 0.03820899501442909, 0.06234658882021904, 0.10919700562953949, 0.0413760244846344, 0.04916759952902794, 0.054400913417339325, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10485509783029556, 0.12122292071580887, 0.0648748055100441, 0.08768714964389801, 0.03434053063392639, 0.01748395338654518, 0.034151818603277206, 0.015289156697690487, 0.023312119767069817, 0.028306512162089348, 0.01872047781944275, 0.028111934661865234, 0.041905295103788376, 0.0209895521402359, 0.04678506404161453, 0.08659638464450836, 0.023631852120161057, 0.024273162707686424, 0.16702400147914886, 0.010438223369419575, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.22901973128318787, 0.05184381827712059, 0.013585173524916172, 0.012337238527834415, 0.018005413934588432, 0.0027703619562089443, 0.042381297796964645, 0.005856257397681475, 0.03614485636353493, 0.13039222359657288, 0.0031534270383417606, 0.01567256823182106, 0.027800394222140312, 0.003554322523996234, 0.07460816204547882, 0.11298283189535141, 0.004272268619388342, 0.006832208018749952, 0.18569746613502502, 0.018073637038469315, 0.005016352981328964, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.18869927525520325, 0.03438714146614075, 0.022344734519720078, 0.019972724840044975, 0.016354026272892952, 0.0068565551191568375, 0.020859038457274437, 0.005695997271686792, 0.03415916487574577, 0.07260986417531967, 0.007857211865484715, 0.018040239810943604, 0.026904456317424774, 0.009020394645631313, 0.06876447051763535, 0.17578734457492828, 0.010720067657530308, 0.00928453542292118, 0.1925639659166336, 0.025180332362651825, 0.012639074586331844, 0.021299341693520546, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1195831298828125, 0.022259412333369255, 0.032947149127721786, 0.020170224830508232, 0.035653311759233475, 0.013459913432598114, 0.017516475170850754, 0.010057873092591763, 0.025856440886855125, 0.05955953523516655, 0.015084508806467056, 0.015008730813860893, 0.053174685686826706, 0.016597608104348183, 0.041555255651474, 0.131293386220932, 0.01929667219519615, 0.015855036675930023, 0.1792508363723755, 0.01618383638560772, 0.02229553461074829, 0.015463392250239849, 0.10187702625989914, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14615531265735626, 0.026727521792054176, 0.01662455126643181, 0.018987685441970825, 0.06278638541698456, 0.015317163430154324, 0.019792240113019943, 0.01422776561230421, 0.025458168238401413, 0.045303549617528915, 0.016364356502890587, 0.037493038922548294, 0.0132886478677392, 0.017496541142463684, 0.0399458184838295, 0.05881758779287338, 0.01926097832620144, 0.024616025388240814, 0.038219697773456573, 0.02157779224216938, 0.02094990760087967, 0.07973217219114304, 0.05017608404159546, 0.17068105936050415, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.11321667581796646, 0.03674636408686638, 0.011786575429141521, 0.010274871252477169, 0.020370664075016975, 0.00524388812482357, 0.015918847173452377, 0.005266785155981779, 0.02489173412322998, 0.06593258678913116, 0.005933663807809353, 0.018209032714366913, 0.021020200103521347, 0.0066674984991550446, 0.034828800708055496, 0.13742128014564514, 0.007927048951387405, 0.008618664927780628, 0.1137719452381134, 0.013557437807321548, 0.009277833625674248, 0.026121344417333603, 0.08499342203140259, 0.19073909521102905, 0.011263742111623287, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.13337711989879608, 0.026216603815555573, 0.03827154263854027, 0.07152578234672546, 0.05317767709493637, 0.013925841078162193, 0.007084188051521778, 0.013450146652758121, 0.009841442108154297, 0.011789779178798199, 0.013537583872675896, 0.038154955953359604, 0.041933052241802216, 0.013882285915315151, 0.03707146272063255, 0.138384610414505, 0.014846334233880043, 0.03156952187418938, 0.05598173290491104, 0.015536684542894363, 0.01595635898411274, 0.045455560088157654, 0.01669965498149395, 0.02532576024532318, 0.03671892359852791, 0.08028541505336761, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10608974099159241, 0.019107727333903313, 0.024468660354614258, 0.027496395632624626, 0.016365809366106987, 0.0050114234909415245, 0.010413105599582195, 0.006081144325435162, 0.005301064345985651, 0.011143166571855545, 0.004565386101603508, 0.01896991953253746, 0.004321120213717222, 0.00481497822329402, 0.02940940298140049, 0.028682034462690353, 0.005097254645079374, 0.0072343479841947556, 0.03412593528628349, 0.010370595380663872, 0.005643266253173351, 0.007283586077392101, 0.029389560222625732, 0.010038800537586212, 0.009134513325989246, 0.546663224697113, 0.012777911499142647, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10467305034399033, 0.03321940079331398, 0.015341250225901604, 0.009373549371957779, 0.026595456525683403, 0.0057878089137375355, 0.013571349903941154, 0.004554884973913431, 0.028058893978595734, 0.026107225567102432, 0.006353434175252914, 0.013315831311047077, 0.026628265157341957, 0.006888872943818569, 0.06204749271273613, 0.05890703946352005, 0.008068050257861614, 0.007557098753750324, 0.0852278545498848, 0.017075754702091217, 0.009256886318325996, 0.019695758819580078, 0.12617813050746918, 0.13061513006687164, 0.011351036839187145, 0.08984372764825821, 0.04638150706887245, 0.007325289770960808, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08561903238296509, 0.021438946947455406, 0.056412599980831146, 0.0570666640996933, 0.019802208989858627, 0.00672749662771821, 0.005809164140373468, 0.004516261164098978, 0.0031647481955587864, 0.01761520653963089, 0.006174598820507526, 0.08767975866794586, 0.012299864552915096, 0.006350455805659294, 0.017522143200039864, 0.14295215904712677, 0.006585489492863417, 0.007875688374042511, 0.030078941956162453, 0.01390750240534544, 0.0073766945861279964, 0.007684784941375256, 0.02216084487736225, 0.012385032139718533, 0.011890063062310219, 0.08669772744178772, 0.19902527332305908, 0.01359447930008173, 0.029586246237158775, 0.0, 0.0, 0.0, 0.0], [0.14064587652683258, 0.013298707082867622, 0.015702949836850166, 0.017357859760522842, 0.02233150415122509, 0.029672270640730858, 0.04172082245349884, 0.01899542473256588, 0.03827710077166557, 0.048635613173246384, 0.03094690665602684, 0.016023898497223854, 0.020880894735455513, 0.032438233494758606, 0.030558118596673012, 0.022808339446783066, 0.03537759929895401, 0.031451594084501266, 0.03497113287448883, 0.01867910660803318, 0.03821907937526703, 0.02257886528968811, 0.06819561868906021, 0.04214095324277878, 0.028620855882763863, 0.03775003179907799, 0.018578052520751953, 0.03376871719956398, 0.03641696274280548, 0.012956855818629265, 0.0, 0.0, 0.0], [0.07168620079755783, 0.06924446672201157, 0.019306905567646027, 0.014161979779601097, 0.016823193058371544, 0.0193806029856205, 0.019257402047514915, 0.0220036581158638, 0.013706518337130547, 0.03578377887606621, 0.018465185537934303, 0.05207168683409691, 0.020085178315639496, 0.01986212469637394, 0.02066212333738804, 0.04725164547562599, 0.021076705306768417, 0.036787249147892, 0.024324089288711548, 0.0038275483530014753, 0.023920675739645958, 0.008533227257430553, 0.026241622865200043, 0.027380064129829407, 0.03461199253797531, 0.022884156554937363, 0.10047907382249832, 0.06913496553897858, 0.025474581867456436, 0.06495600938796997, 0.03061538189649582, 0.0, 0.0], [0.0691077709197998, 0.0370122492313385, 0.03862115740776062, 0.05933326855301857, 0.015923550352454185, 0.007918563671410084, 0.010371048003435135, 0.0066156634129583836, 0.002520075300708413, 0.026019366458058357, 0.007905230857431889, 0.02965201810002327, 0.04000623896718025, 0.008451344445347786, 0.010741154663264751, 0.05027563124895096, 0.009428859688341618, 0.013601033948361874, 0.05036921799182892, 0.03176714852452278, 0.010793027468025684, 0.007216801401227713, 0.006478427443653345, 0.01480061188340187, 0.021585967391729355, 0.15769484639167786, 0.08884759992361069, 0.0190169308334589, 0.02972934953868389, 0.03316137194633484, 0.050883661955595016, 0.034150850027799606, 0.0], [0.14375509321689606, 0.01681104488670826, 0.009386662393808365, 0.006830313708633184, 0.011656845919787884, 0.0015672279987484217, 0.019711481407284737, 0.002398042706772685, 0.021235886961221695, 0.04683680087327957, 0.0016905148513615131, 0.0058271917514503, 0.011979999952018261, 0.0018251396249979734, 0.042313333600759506, 0.054913729429244995, 0.0021786566358059645, 0.0024170097894966602, 0.09604065865278244, 0.005752880126237869, 0.0025577889755368233, 0.0071212234906852245, 0.08889931440353394, 0.10852081328630447, 0.005179054103791714, 0.03657734394073486, 0.02471994049847126, 0.0037347625475376844, 0.031077096238732338, 0.016887947916984558, 0.09450862556695938, 0.07171519845724106, 0.0033723204396665096]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0004246663593221456, 0.9995753169059753, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0005621908348985016, 0.016407281160354614, 0.9830306172370911, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0011627554195001721, 0.021681973710656166, 0.0037620372604578733, 0.9733933210372925, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [3.7244368286337703e-05, 0.00017202268645633012, 0.0002814392792060971, 0.0027421435806900263, 0.9967671632766724, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00826845783740282, 0.00023985625011846423, 7.361917960224673e-05, 6.43773382762447e-05, 0.0001756635756464675, 0.9911779761314392, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0012215041788294911, 0.005400444380939007, 0.0016716312384232879, 0.00040775578236207366, 0.0006163655198179185, 0.001093117636628449, 0.989589273929596, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0012459794525057077, 0.0009121228358708322, 0.0005976726533845067, 0.00013656896771863103, 0.00033041107235476375, 0.001572280889376998, 0.0038808276876807213, 0.9913240671157837, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00028217516955919564, 0.004068182315677404, 0.0026605194434523582, 0.0013093105517327785, 0.008030476048588753, 0.00028790938085876405, 0.00022922940843272954, 0.0003948427038267255, 0.9827372431755066, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [3.47392306139227e-05, 0.0003953832492697984, 0.00013272685464471579, 0.0002585221081972122, 0.001085555530153215, 9.198026964440942e-05, 0.000326707202475518, 0.000542744412086904, 0.006105918437242508, 0.9910256266593933, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0033785062842071056, 5.0908744015032426e-05, 1.6452069758088328e-05, 1.69261602422921e-05, 4.18141353293322e-05, 0.49394020438194275, 0.00012981283362023532, 0.0008837337954901159, 3.221205042791553e-05, 2.7252061045146547e-05, 0.5014821887016296, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [8.416055788984522e-05, 0.0013417234877124429, 0.001261359779164195, 0.0021450764033943415, 0.004042366519570351, 0.00048305385280400515, 0.0001158266604761593, 0.00015203595103230327, 2.6925330530502833e-05, 0.00012675137259066105, 0.00031289938488043845, 0.9899077415466309, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0003782230196520686, 0.0009837561519816518, 0.03934125974774361, 0.002732245484367013, 0.0036680190823972225, 0.00011039189848816022, 0.00012931032688356936, 0.00021743457182310522, 0.00010623293928802013, 0.0007748182397335768, 6.647665577474982e-05, 0.0003148667747154832, 0.9511768817901611, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0021017189137637615, 2.443686025799252e-05, 7.78878529672511e-06, 8.651618372823577e-06, 2.014001256611664e-05, 0.2997134327888489, 7.525274122599512e-05, 0.0004898307379335165, 1.845947736001108e-05, 1.534453986096196e-05, 0.32833898067474365, 4.175798676442355e-05, 6.469185791502241e-06, 0.3691376745700836, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0001496832846896723, 0.00011296618322376162, 0.00036294886376708746, 0.00018591222760733217, 0.00016460890765301883, 4.1432296711718664e-05, 2.8764745366061106e-05, 7.786958303768188e-05, 0.0009200992644764483, 0.0103401904925704, 2.7572339604375884e-05, 1.7833237507147714e-05, 0.0003305452992208302, 2.4375704015255906e-05, 0.9872152805328369, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00010753834794741124, 0.0021781916730105877, 0.002042611828073859, 0.004251930397003889, 0.006989913992583752, 2.511881393729709e-05, 0.0007779046427458525, 0.0005783551605418324, 0.00293784704990685, 0.033225417137145996, 1.71992760442663e-05, 0.0008936497615650296, 0.0015238587511703372, 1.4656818166258745e-05, 0.00622264388948679, 0.9382131695747375, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.001363488263450563, 1.3226085684436839e-05, 4.013950274384115e-06, 4.803066076419782e-06, 1.0257452231599018e-05, 0.19665955007076263, 4.527267810772173e-05, 0.0002776262117549777, 1.1714444553945214e-05, 9.473311365582049e-06, 0.22919613122940063, 2.6430649086250924e-05, 4.101847935089609e-06, 0.26576095819473267, 8.515355148119852e-06, 4.536029791779583e-06, 0.30659976601600647, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0008924751309677958, 0.00013490811397787184, 4.779841037816368e-05, 5.8037036069436e-05, 0.00010480164928594604, 0.012799090705811977, 0.0007168247830122709, 0.032579511404037476, 2.6449959477758966e-05, 0.00011185064795427024, 0.011884261853992939, 4.010266638942994e-05, 5.555404641199857e-05, 0.012377863749861717, 0.00010783471225295216, 5.4043663112679496e-05, 0.013122126460075378, 0.914886474609375, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [5.075309672974981e-06, 4.396129952510819e-05, 3.398501576157287e-05, 7.940301293274388e-05, 5.477911327034235e-05, 7.921535711830074e-07, 9.313323971582577e-06, 7.727071533736307e-06, 8.597254054620862e-05, 0.00012274031178094447, 5.141479277881444e-07, 1.702793611002562e-06, 3.83417536795605e-05, 4.4509667418424215e-07, 0.00013928221596870571, 0.0003275797644164413, 3.9947968843989656e-07, 3.948126504838001e-06, 0.999043881893158, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [9.346180740976706e-05, 0.0018395738443359733, 0.0025233125779777765, 0.018087238073349, 0.0029363748617470264, 0.00027335810591466725, 4.872974750469439e-05, 0.0004212774510961026, 0.00015624347724951804, 0.0009748333832249045, 0.00020533586211968213, 0.0010228825267404318, 0.0019548034761101007, 0.0001947036653291434, 0.0011294205905869603, 0.0016656103543937206, 0.00018734042532742023, 0.0009503448964096606, 0.0004455175076145679, 0.964889645576477, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0010859702015295625, 8.512333806720562e-06, 2.5307288069598144e-06, 3.0625205909018405e-06, 5.975129170110449e-06, 0.13292615115642548, 3.345326331327669e-05, 0.00018891248328145593, 8.477756637148559e-06, 6.540415142808342e-06, 0.1644592434167862, 1.813003655115608e-05, 3.0627427349827485e-06, 0.19719170033931732, 6.4298355937353335e-06, 3.444739604674396e-06, 0.23317119479179382, 0.0022796259727329016, 3.713432988661225e-06, 3.52880269929301e-05, 0.2685585916042328, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0004197956877760589, 0.00011805014219135046, 0.00014240501332096756, 3.796788223553449e-05, 0.00019043161591980606, 0.00176512252073735, 0.00057098304387182, 0.0005008855368942022, 8.840763621265069e-05, 0.0001420867774868384, 0.001663984265178442, 3.348111204104498e-05, 2.4413982828264125e-05, 0.00175465049687773, 6.520311580970883e-05, 2.414266964478884e-05, 0.0018299149814993143, 0.0015691040316596627, 3.974881110480055e-05, 0.00015712414460722357, 0.0018554475391283631, 0.9870065450668335, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [7.501777145080268e-05, 0.0014379826607182622, 6.345151632558554e-05, 0.00010864829528145492, 0.00015633167640771717, 3.2101231681735953e-06, 0.002203279873356223, 0.00022076554887462407, 5.240297468844801e-05, 4.8815789341460913e-05, 2.2644592263532104e-06, 1.5327233995776623e-05, 4.157140665483894e-06, 2.022839225901407e-06, 6.296796982496744e-06, 4.848669050261378e-05, 1.9928991150663933e-06, 3.247004497097805e-05, 0.0012695090845227242, 1.9632627299870364e-05, 1.8090934190695407e-06, 0.0005810288130305707, 0.9936450719833374, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [7.450151315424591e-05, 0.0006139386096037924, 0.0009361191187053919, 0.0008487795130349696, 0.002850631484761834, 1.0365051821281668e-05, 0.00021614256547763944, 0.00017397530609741807, 0.0020508402958512306, 0.005805297289043665, 8.055229955061805e-06, 8.086592424660921e-05, 0.0007702436414547265, 7.28818440620671e-06, 0.0010576159693300724, 0.002275596372783184, 6.663255135208601e-06, 0.00011621018347796053, 0.0005972451181150973, 8.736297604627907e-05, 6.332331849989714e-06, 6.096452852943912e-05, 6.090577517170459e-05, 0.9812840819358826, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0005848742439411581, 0.00015909200010355562, 1.083647021005163e-05, 7.365475903498009e-05, 0.00011349534906912595, 0.0008256662986241281, 0.00031911072437651455, 0.018529996275901794, 1.0226598533336073e-05, 4.9587219109525904e-05, 0.0007716424879617989, 4.454819281818345e-05, 9.865034371614456e-06, 0.0008067170856520534, 2.2673882995150052e-05, 1.2464139217627235e-05, 0.0008449103333987296, 0.008790111169219017, 3.5792407288681716e-05, 3.66286258213222e-05, 0.0008917527738958597, 0.0010791773675009608, 0.0003708462754730135, 0.0001083713723346591, 0.9654979109764099, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [1.6459925973322242e-05, 0.00017152438522316515, 3.2083211408462375e-05, 0.00010234172805212438, 0.002631882205605507, 9.886184670904186e-06, 3.250848749303259e-05, 3.7417539715534076e-05, 0.00012631539721041918, 4.9912112444872037e-05, 8.302540663862601e-06, 8.443430851912126e-05, 3.127968739136122e-05, 7.633363566128537e-06, 1.0101352927449625e-05, 5.6673809012863785e-05, 7.442129117407603e-06, 2.7689655325957574e-05, 1.841835728555452e-05, 2.879437261071871e-06, 6.840427886345424e-06, 4.279875156498747e-06, 0.00043176551116630435, 0.0001761750172590837, 8.995599637273699e-05, 0.9958257675170898, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [2.6368099497631192e-05, 0.00018008743063546717, 0.0001808296947274357, 0.00030464722658507526, 0.00039390946039929986, 4.674849333241582e-05, 2.7231719286646694e-05, 4.8734953452367336e-05, 0.00029131697374396026, 0.0004206168814562261, 3.804643711191602e-05, 0.0002524516312405467, 5.6067383411573246e-05, 3.82037615054287e-05, 0.0015365129802376032, 0.001253720954991877, 3.593428846215829e-05, 2.303666406078264e-05, 0.0001803626073524356, 0.0001226610183948651, 3.517790537443943e-05, 6.924685294507071e-05, 0.00011267305671935901, 0.0008507389575242996, 0.0001436190214008093, 0.00023528003657702357, 0.9930958151817322, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0007048255065456033, 5.675383727066219e-05, 3.351289251440903e-06, 7.472657671314664e-06, 1.785946005838923e-05, 0.0008156524272635579, 4.9529528041603044e-05, 0.0013530971482396126, 3.741759792319499e-05, 0.00014378006744664162, 0.0007639332907274365, 9.991676051868126e-06, 2.8844410735473502e-06, 0.0007552222232334316, 0.00010159592784475535, 3.1205443065118743e-06, 0.000806094438303262, 0.0011406756239011884, 1.4343090697366279e-05, 9.910167136695236e-06, 0.0008649186929687858, 8.663265907671303e-05, 3.4695403883233666e-05, 0.00010265821038046852, 0.007663471158593893, 4.187891317997128e-05, 3.4908698580693454e-06, 0.9844048619270325, 0.0, 0.0, 0.0, 0.0, 0.0], [2.0288332962081768e-05, 2.954272713395767e-05, 5.0375034334138036e-05, 0.00097783247474581, 0.3728252649307251, 6.6613192757358775e-06, 1.573364352225326e-05, 3.981243571615778e-05, 0.00022353240638040006, 0.00012674322351813316, 5.118445187690668e-06, 0.00024116165877785534, 1.2973839147889521e-05, 4.8006963879743125e-06, 2.3321663320530206e-05, 7.723527232883498e-05, 4.59344209957635e-06, 1.9647124645416625e-05, 0.00021129964443389326, 1.1453522347437683e-05, 4.380489826871781e-06, 1.4442671272263397e-05, 3.676743290270679e-05, 0.00011845333210658282, 3.7977195461280644e-05, 0.0007802762556821108, 0.00040483634802512825, 1.042955591401551e-05, 0.6236649751663208, 0.0, 0.0, 0.0, 0.0], [2.0582927390933037e-05, 8.831251761876047e-05, 0.00020454880723264068, 0.0003018901334144175, 8.223887562053278e-05, 1.9704415535670705e-05, 0.00014085126167628914, 2.896317164413631e-05, 7.669385013286956e-06, 3.7246169085847214e-05, 1.673677070357371e-05, 6.404696614481509e-05, 0.0006910591037012637, 1.6027584933908656e-05, 0.00015603694191668183, 0.00014825927792117, 1.5700084986747243e-05, 9.155390580417588e-05, 8.525098382961005e-05, 4.904517481918447e-06, 1.5784396964590997e-05, 5.293117283144966e-05, 0.000529835931956768, 0.0005658396985381842, 6.167324318084866e-05, 6.729497545165941e-05, 0.0003077442815992981, 1.0369159099354874e-05, 6.799342372687533e-05, 0.9960988759994507, 0.0, 0.0, 0.0], [1.3371318345889449e-05, 0.0009821535786613822, 0.0004154810740146786, 0.0001144233756349422, 0.00038730789674445987, 5.660860551870428e-06, 0.0012746280990540981, 0.0005708065000362694, 0.0006383630097843707, 0.0005776663310825825, 4.127733518544119e-06, 9.16175213205861e-06, 9.142841736320406e-05, 3.774867764150258e-06, 1.3575321645475924e-05, 0.0002916179655585438, 3.4474890071578557e-06, 7.899192860350013e-05, 0.003189122537150979, 4.885083853878314e-06, 3.165286670991918e-06, 1.4087469025980681e-05, 0.0001567144354339689, 0.0003544889041222632, 0.00017265471979044378, 0.0013050634879618883, 0.00021867688337806612, 2.6776719096233137e-05, 0.00026460207300260663, 1.3334529285202734e-05, 0.9888005256652832, 0.0, 0.0], [2.745506208157167e-05, 0.00016438262537121773, 7.99642366473563e-05, 0.001191496616229415, 0.0007883630460128188, 2.658414359757444e-06, 3.005756479979027e-05, 7.4579688771336805e-06, 0.00014940995606593788, 2.8857133656856604e-05, 1.873827500276093e-06, 0.00033288003760389984, 5.160855653230101e-05, 1.7577448261363315e-06, 0.0001265659084310755, 0.00014267012011259794, 1.6954454622464254e-06, 2.1952573661110364e-05, 0.00023040804080665112, 4.429338878253475e-05, 1.6103306279546814e-06, 2.7008050892618485e-05, 0.00023884844267740846, 0.00019046018132939935, 9.496136954112444e-06, 0.000446643796749413, 0.00022095769236329943, 5.379170943342615e-06, 0.0006956399301998317, 0.0001547076681163162, 0.0002548544143792242, 0.9943286776542664, 0.0], [0.006231049541383982, 9.72282505244948e-05, 6.871603090985445e-06, 2.1151156033738516e-05, 5.8280591474613175e-05, 0.007238905411213636, 2.0987936295568943e-05, 0.00025459096650592983, 6.243876850930974e-05, 2.0924684577039443e-05, 0.007872147485613823, 5.5853353842394426e-05, 9.868796041700989e-06, 0.009169402532279491, 7.203016866696998e-05, 7.068700597301358e-06, 0.010345976799726486, 0.0013096537441015244, 3.803684376180172e-05, 8.022711699595675e-05, 0.012053261511027813, 4.071998773724772e-05, 3.6860749332845444e-06, 3.4713455534074455e-05, 0.0005061827832832932, 8.918941603042185e-05, 2.9112143238307908e-05, 0.0012772815534844995, 8.489656465826556e-05, 0.00018447409092914313, 0.00013425754150375724, 6.813067739130929e-05, 0.9425214529037476]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.943029522895813, 0.05697042867541313, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9185556173324585, 0.03280005604028702, 0.048644352704286575, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8779287934303284, 0.056434255093336105, 0.04271192103624344, 0.02292500250041485, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.804131805896759, 0.029098201543092728, 0.075567327439785, 0.05643591657280922, 0.03476677089929581, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4943104684352875, 0.02018355205655098, 0.027966560795903206, 0.01831907220184803, 0.03144203498959541, 0.40777823328971863, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6057478785514832, 0.02924242615699768, 0.09491512179374695, 0.07609347254037857, 0.06614658236503601, 0.08705786615610123, 0.0407966710627079, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4483822286128998, 0.04542430862784386, 0.0740148276090622, 0.06864846497774124, 0.09376619011163712, 0.0877426490187645, 0.06534270942211151, 0.11667861044406891, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.49169260263442993, 0.13782073557376862, 0.03955018147826195, 0.06153320148587227, 0.04539964720606804, 0.04073145240545273, 0.06228705495595932, 0.05861866846680641, 0.062366463243961334, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5404124855995178, 0.044426653534173965, 0.03957854211330414, 0.04188809171319008, 0.07529856264591217, 0.04669506475329399, 0.048475231975317, 0.05500520393252373, 0.08293062448501587, 0.02528950944542885, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.28273555636405945, 0.014234175905585289, 0.017647748813033104, 0.011433064006268978, 0.021741706877946854, 0.2666538953781128, 0.015403537079691887, 0.047349266707897186, 0.017767544835805893, 0.013926065526902676, 0.2911074459552765, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.34496626257896423, 0.046116773039102554, 0.05771546810865402, 0.11131857335567474, 0.11289366334676743, 0.027930336073040962, 0.0385919027030468, 0.05656527727842331, 0.05864058807492256, 0.06648595631122589, 0.026114359498023987, 0.052660852670669556, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.46992170810699463, 0.032015684992074966, 0.1077289879322052, 0.027006765827536583, 0.04465881362557411, 0.022773560136556625, 0.0231170691549778, 0.025491517037153244, 0.049502693116664886, 0.026573937386274338, 0.01970885880291462, 0.06337957829236984, 0.08812081813812256, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.20661765336990356, 0.01037929393351078, 0.012261927127838135, 0.008319098502397537, 0.016007069498300552, 0.19527742266654968, 0.011453290469944477, 0.03475669026374817, 0.01307358592748642, 0.010938976891338825, 0.21602483093738556, 0.005866494961082935, 0.023642312735319138, 0.23538129031658173, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.33993491530418396, 0.029161635786294937, 0.09540709853172302, 0.03395187482237816, 0.08440458029508591, 0.012559544295072556, 0.02935866080224514, 0.024564165621995926, 0.10622433573007584, 0.04689216986298561, 0.011469592340290546, 0.006369193084537983, 0.11145274341106415, 0.011317990720272064, 0.0569315105676651, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4040880799293518, 0.024195658043026924, 0.038910072296857834, 0.014727434143424034, 0.024456558749079704, 0.038450006395578384, 0.03923036903142929, 0.037171367555856705, 0.06030004844069481, 0.041985444724559784, 0.037167176604270935, 0.01639123260974884, 0.0392896942794323, 0.03772977367043495, 0.13448578119277954, 0.011421292088925838, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15876448154449463, 0.008203906007111073, 0.0092952661216259, 0.0062217190861701965, 0.011793204583227634, 0.15069791674613953, 0.008851958438754082, 0.026313532143831253, 0.010186923667788506, 0.00843372568488121, 0.16762229800224304, 0.004420033656060696, 0.01821179874241352, 0.183508038520813, 0.020927347242832184, 0.006447790190577507, 0.20010006427764893, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.17539216578006744, 0.02056964486837387, 0.01829143613576889, 0.009298020042479038, 0.017377890646457672, 0.042534783482551575, 0.02070159651339054, 0.050443943589925766, 0.02543804794549942, 0.017218226566910744, 0.04311535507440567, 0.013349834829568863, 0.0285286046564579, 0.045972540974617004, 0.03408820927143097, 0.019834214821457863, 0.04992839694023132, 0.36791715025901794, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.26033347845077515, 0.017148371785879135, 0.03745276480913162, 0.07594798505306244, 0.04674699530005455, 0.018068527802824974, 0.031346458941698074, 0.037415146827697754, 0.07175809890031815, 0.05872536823153496, 0.0170787014067173, 0.04030593857169151, 0.05706355720758438, 0.01711357943713665, 0.1049133837223053, 0.046705346554517746, 0.017230207100510597, 0.024682143703103065, 0.019963975995779037, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.30547064542770386, 0.05190496891736984, 0.043468981981277466, 0.02184685505926609, 0.02101718634366989, 0.03390473872423172, 0.04190470278263092, 0.03909286484122276, 0.028871973976492882, 0.023003432899713516, 0.03205801919102669, 0.023334519937634468, 0.0711059421300888, 0.03290088474750519, 0.061641961336135864, 0.03183262050151825, 0.033767636865377426, 0.04571487754583359, 0.035015225410461426, 0.022141898050904274, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1224295049905777, 0.006105300039052963, 0.006670300383120775, 0.00458177737891674, 0.009337784722447395, 0.11381794512271881, 0.006783359684050083, 0.01971990056335926, 0.00758085772395134, 0.006613647099584341, 0.12765924632549286, 0.0035026604309678078, 0.01400233618915081, 0.14000985026359558, 0.015684885904192924, 0.005092614330351353, 0.15336214005947113, 0.03527415543794632, 0.02246563322842121, 0.006954458076506853, 0.17235167324543, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.20538486540317535, 0.03477850183844566, 0.014682911336421967, 0.03118330053985119, 0.030931269749999046, 0.021952766925096512, 0.03290865570306778, 0.05740531533956528, 0.05587516725063324, 0.04864277318120003, 0.02352074719965458, 0.015108555555343628, 0.02738633006811142, 0.02451845072209835, 0.060604583472013474, 0.034776147454977036, 0.026137804612517357, 0.051684360951185226, 0.06281405687332153, 0.020291464403271675, 0.02860172651708126, 0.0908101499080658, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.23978599905967712, 0.031323499977588654, 0.05037755146622658, 0.01586942747235298, 0.03901456296443939, 0.02280554547905922, 0.04285356029868126, 0.028882304206490517, 0.04046262055635452, 0.0341072604060173, 0.022644521668553352, 0.03923071548342705, 0.0723857581615448, 0.022345518693327904, 0.04958106949925423, 0.03193334490060806, 0.023325597867369652, 0.04521360620856285, 0.030554356053471565, 0.022876497358083725, 0.02473229542374611, 0.055494822561740875, 0.014199569821357727, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.23581692576408386, 0.020558306947350502, 0.04375005513429642, 0.02970486506819725, 0.03703876957297325, 0.014953454956412315, 0.04004311189055443, 0.027184365317225456, 0.0457618422806263, 0.03809259459376335, 0.014181883074343204, 0.03789152577519417, 0.06518243253231049, 0.014182924292981625, 0.05489495024085045, 0.023720961064100266, 0.014592787250876427, 0.025570029392838478, 0.07356183975934982, 0.039182331413030624, 0.014925522729754448, 0.04628865420818329, 0.027801064774394035, 0.015118853189051151, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14254964888095856, 0.012010160833597183, 0.016881300136446953, 0.020742561668157578, 0.032451752573251724, 0.029625998809933662, 0.030295221135020256, 0.05620869994163513, 0.029608748853206635, 0.02948184125125408, 0.03263983502984047, 0.010038234293460846, 0.04078620672225952, 0.03462785482406616, 0.03391636535525322, 0.02015574835240841, 0.036843158304691315, 0.0606469102203846, 0.047447122633457184, 0.032525286078453064, 0.040400370955467224, 0.05947763845324516, 0.03129400685429573, 0.04792547971010208, 0.07141980528831482, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2222907692193985, 0.059111371636390686, 0.03702675551176071, 0.04059014096856117, 0.0272544976323843, 0.019174763932824135, 0.03171538934111595, 0.02046213671565056, 0.038113273680210114, 0.019927890971302986, 0.018538912758231163, 0.015436704270541668, 0.04536491632461548, 0.01935577020049095, 0.050358302891254425, 0.03328138589859009, 0.02017974853515625, 0.03679436072707176, 0.04331387206912041, 0.028476405888795853, 0.02131732925772667, 0.047712888568639755, 0.013107037171721458, 0.02633604407310486, 0.030211349949240685, 0.03454800695180893, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.28438133001327515, 0.023870104923844337, 0.04641882702708244, 0.010260095819830894, 0.03390985727310181, 0.01830146834254265, 0.0237088892608881, 0.0247668344527483, 0.025767752900719643, 0.022968804463744164, 0.016735462471842766, 0.013406947255134583, 0.045986607670784, 0.01667322963476181, 0.08106391876935959, 0.05033260956406593, 0.016726160421967506, 0.019904790446162224, 0.03032534383237362, 0.01014632172882557, 0.01731823943555355, 0.019040953367948532, 0.01108111348003149, 0.052046384662389755, 0.033353518694639206, 0.03880883380770683, 0.012695519253611565, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.11854357272386551, 0.01335094589740038, 0.013422034680843353, 0.030273202806711197, 0.02616293355822563, 0.020776210352778435, 0.021280204877257347, 0.03798932954668999, 0.03536444157361984, 0.036651790142059326, 0.022004006430506706, 0.01538003608584404, 0.030023131519556046, 0.023299671709537506, 0.03000757284462452, 0.014861369505524635, 0.02452193759381771, 0.03999800235033035, 0.03900156170129776, 0.036007679998874664, 0.026851212605834007, 0.06795871257781982, 0.038931310176849365, 0.057258062064647675, 0.0665983259677887, 0.03075685352087021, 0.023343220353126526, 0.059382691979408264, 0.0, 0.0, 0.0, 0.0, 0.0], [0.25091853737831116, 0.01135188713669777, 0.023033952340483665, 0.01884976588189602, 0.013208821415901184, 0.016430404037237167, 0.03763698413968086, 0.021519413217902184, 0.03823148459196091, 0.03122309409081936, 0.01608109660446644, 0.017179012298583984, 0.0823964774608612, 0.01577541045844555, 0.04969313368201256, 0.033979032188653946, 0.016469145193696022, 0.025656109675765038, 0.053269319236278534, 0.024380972608923912, 0.017032906413078308, 0.031062550842761993, 0.014834268018603325, 0.04310780018568039, 0.027818351984024048, 0.015127941034734249, 0.012749841436743736, 0.026852674782276154, 0.014129597693681717, 0.0, 0.0, 0.0, 0.0], [0.19256946444511414, 0.022833675146102905, 0.01449588406831026, 0.028055010363459587, 0.03290426358580589, 0.018577100709080696, 0.023776723071932793, 0.014988909475505352, 0.027755476534366608, 0.01995212957262993, 0.018426887691020966, 0.0268084853887558, 0.04026195779442787, 0.01895732805132866, 0.01998024806380272, 0.039050325751304626, 0.01949433982372284, 0.030714301392436028, 0.07932323962450027, 0.03619766980409622, 0.02037958987057209, 0.023319289088249207, 0.018723847344517708, 0.05692768841981888, 0.023927535861730576, 0.03924323245882988, 0.02178351581096649, 0.020374197512865067, 0.04178833216428757, 0.00840924121439457, 0.0, 0.0, 0.0], [0.1891939342021942, 0.01435206737369299, 0.027829350903630257, 0.0189360398799181, 0.05455230921506882, 0.024302387610077858, 0.0207525584846735, 0.03050178475677967, 0.01690041646361351, 0.029904041439294815, 0.023111775517463684, 0.02166486158967018, 0.03335980698466301, 0.023050500079989433, 0.02792702615261078, 0.026253484189510345, 0.0242579597979784, 0.024394970387220383, 0.017666669562458992, 0.02208106406033039, 0.025441503152251244, 0.023996729403734207, 0.015941407531499863, 0.021863479167222977, 0.04876275360584259, 0.01023405697196722, 0.025762980803847313, 0.06192327290773392, 0.06737110018730164, 0.01818411611020565, 0.009525515139102936, 0.0, 0.0], [0.27111950516700745, 0.05460572615265846, 0.039705973118543625, 0.03512895852327347, 0.02031376212835312, 0.0081838583573699, 0.022556638345122337, 0.01403643935918808, 0.02619846910238266, 0.032355356961488724, 0.007185023743659258, 0.010813490487635136, 0.05162610113620758, 0.007057540584355593, 0.04232776537537575, 0.01850598305463791, 0.006952997762709856, 0.012550847604870796, 0.037855181843042374, 0.014787373133003712, 0.007130768615752459, 0.01895672082901001, 0.01075897179543972, 0.023540528491139412, 0.01625620573759079, 0.007915404625236988, 0.03652312234044075, 0.014562270604074001, 0.019251469522714615, 0.010911807417869568, 0.0343967042863369, 0.06592899560928345, 0.0], [0.14836537837982178, 0.014506888575851917, 0.007646295242011547, 0.011038758791983128, 0.02630247175693512, 0.01319483295083046, 0.027184898033738136, 0.017392203211784363, 0.016122139990329742, 0.032880790531635284, 0.014050280675292015, 0.0071673463098704815, 0.016120055690407753, 0.014647023752331734, 0.018450895324349403, 0.013102126307785511, 0.01528322696685791, 0.01454298384487629, 0.05961218476295471, 0.02238387055695057, 0.016796991229057312, 0.10928259789943695, 0.07264000922441483, 0.08393709361553192, 0.025248989462852478, 0.03774847462773323, 0.016226941719651222, 0.024953871965408325, 0.042165856808423996, 0.004525808617472649, 0.014491353183984756, 0.01914571039378643, 0.022841647267341614]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09646990895271301, 0.9035300612449646, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04325238987803459, 0.08177754282951355, 0.8749701380729675, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09995394200086594, 0.025312727317214012, 0.02010803483426571, 0.8546252846717834, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.024889368563890457, 0.00320735527202487, 0.0018421593122184277, 0.022361503913998604, 0.9476996660232544, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10732383280992508, 0.017841672524809837, 0.01955333538353443, 0.04333319514989853, 0.10211498290300369, 0.7098329067230225, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.006426361855119467, 0.00044797800364904106, 0.00014756589371245354, 0.00046936701983213425, 0.0014411886222660542, 0.003859696676954627, 0.98720782995224, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00103545852471143, 0.00019901820633094758, 0.00016020411567296833, 6.937271973583847e-05, 0.00038674141978845, 0.005171590484678745, 0.8964057564735413, 0.0965719074010849, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0012883321614935994, 0.0003233453317079693, 0.00026527189766056836, 0.00025490616098977625, 0.000201298258616589, 0.00010049015691038221, 0.0005700886249542236, 0.000409130152547732, 0.9965871572494507, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0001874157169368118, 1.6196530850720592e-05, 4.2281039895897266e-06, 0.000287588540231809, 1.1125704986625351e-05, 9.805656191019807e-06, 0.0001556719362270087, 7.632971392013133e-05, 0.0034869094379246235, 0.9957647323608398, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.015741512179374695, 0.0006393603980541229, 0.00045715278247371316, 0.0009912519017234445, 0.0021140354219824076, 0.0180897768586874, 0.0471203438937664, 0.07010912150144577, 0.06152832508087158, 0.27690398693084717, 0.5063051581382751, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0006166594685055315, 0.0005207078065723181, 4.61151976196561e-05, 0.00146130018401891, 0.0005623754695989192, 4.4476037146523595e-05, 0.00036539664142765105, 0.0002860166132450104, 0.004506127443164587, 0.005816521123051643, 0.0007244544103741646, 0.9850499033927917, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0010478557087481022, 3.062814721488394e-05, 0.00017344979278277606, 5.423119000624865e-05, 6.388417386915535e-05, 1.1261126928729936e-05, 1.7169008060591295e-05, 1.3931307876191568e-05, 0.0020760390907526016, 0.00026926834834739566, 0.00015268517017830163, 0.0036844322457909584, 0.992405116558075, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.010746272280812263, 0.00025354334502480924, 0.0001532292371848598, 0.00029287015786394477, 0.0005376793560571969, 0.004423544742166996, 0.00988433975726366, 0.012843050062656403, 0.012738612480461597, 0.05966852977871895, 0.107729971408844, 0.024745700880885124, 0.07808694988489151, 0.6778956651687622, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00041218000114895403, 2.6218738639727235e-05, 1.7755904991645366e-05, 0.00019181883544661105, 3.2979492061713245e-06, 3.912682132067857e-06, 1.043809788825456e-05, 4.9068494263337925e-06, 0.0005868570297025144, 0.003038151189684868, 3.693124745041132e-05, 0.0007724304450675845, 0.009622604586184025, 0.00016094396414700896, 0.9851114153862, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00010886583186220378, 3.5845487218466587e-06, 5.191657237446634e-06, 3.083837145823054e-05, 1.8408762116450816e-05, 7.954765806061914e-07, 3.3532073757669423e-06, 6.574365215783473e-06, 0.0007270933128893375, 0.0018232465954497457, 7.853259376133792e-06, 0.0003062748000957072, 0.006975048687309027, 3.4850869269575924e-05, 0.0282927006483078, 0.9616552591323853, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.006073986645787954, 8.815194451017305e-05, 4.5837816287530586e-05, 8.250484825111926e-05, 0.0001272043155040592, 0.0009312513284385204, 0.0016699967673048377, 0.001938193803653121, 0.002101697726175189, 0.010131671093404293, 0.016047311946749687, 0.003835388459265232, 0.012242135591804981, 0.10037057846784592, 0.07853133976459503, 0.10997355729341507, 0.655809223651886, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0021862962748855352, 1.1892905604327098e-05, 1.673595943429973e-05, 2.081541060761083e-05, 2.116251016559545e-05, 0.000771492428611964, 0.0008652104297652841, 0.0005560291465371847, 0.0001565588463563472, 0.0019499900517985225, 0.010891195386648178, 0.0003954307467211038, 0.0015144682256504893, 0.06916307657957077, 0.00430810172110796, 0.004035161342471838, 0.4939703941345215, 0.40916600823402405, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [1.979802800633479e-05, 4.972388865098765e-07, 9.16571263331889e-09, 1.3230416584519844e-07, 5.6018219396491986e-08, 1.1903876995233986e-08, 6.153105118755775e-07, 4.0685506519366754e-08, 4.517455181485275e-06, 1.4017791727383155e-05, 1.6456915830076468e-07, 1.086384372683824e-06, 7.44406133890152e-06, 1.005667400022503e-06, 2.75950224022381e-05, 0.00031378038693219423, 6.201125415827846e-06, 8.213370165321976e-06, 0.9995948672294617, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [6.497167760244338e-06, 8.692019264344708e-07, 1.4051378229851252e-06, 3.7704828059759166e-07, 3.3291723866568645e-07, 6.104961869368708e-08, 2.542995503063139e-08, 1.1067748317827864e-07, 3.670873775263317e-05, 5.704808927475824e-07, 6.172335247356386e-07, 0.00015627949323970824, 0.00014737885794602334, 3.010375849044067e-06, 6.130654219305143e-05, 0.0017769918777048588, 1.8545919374446385e-05, 3.426316106924787e-05, 0.923351526260376, 0.07440318167209625, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.003649001009762287, 3.7494322896236554e-05, 1.74708020495018e-05, 2.8449197998270392e-05, 3.312424087198451e-05, 0.00019569217693060637, 0.0002647593501023948, 0.0002469043538440019, 0.00028033158741891384, 0.0013949184212833643, 0.0017910305177792907, 0.00042468024184927344, 0.001462370972149074, 0.009717755019664764, 0.008053947240114212, 0.01320058386772871, 0.06315966695547104, 0.07152792811393738, 0.07523112744092941, 0.024205094203352928, 0.7250776290893555, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.000641089805867523, 3.925701548723737e-06, 1.1607372698563267e-06, 1.537548200758465e-06, 9.072650755115319e-07, 8.05819763627369e-06, 1.3223971109255217e-05, 4.6199284042813815e-06, 3.678836583276279e-05, 4.5651795517187566e-05, 7.577823998872191e-05, 2.274175494676456e-05, 5.6684530136408284e-05, 0.0004737511626444757, 0.0003640766954049468, 0.00042400247184559703, 0.00375733501277864, 0.004363126587122679, 0.007395228371024132, 0.005022629629820585, 0.05575620383024216, 0.9215314984321594, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0005377684719860554, 8.12255109394755e-07, 2.3018603769742185e-06, 1.470038341722102e-06, 1.8703678961173864e-06, 6.02451621034561e-07, 4.12783674619277e-06, 1.6705761254343088e-06, 2.047191856036079e-06, 7.430045661749318e-05, 3.5942291560786543e-06, 7.171521247073542e-06, 3.052820102311671e-05, 1.8927734345197678e-05, 0.0002543879672884941, 0.0001949639990925789, 0.00012626573152374476, 0.00018230534624308348, 0.0037352300714701414, 0.0006727128056809306, 0.0015798343811184168, 0.011177700012922287, 0.9813893437385559, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [8.512814383720979e-05, 4.870869929618493e-07, 1.2458942819648655e-06, 2.0811235401652084e-07, 1.92428601053507e-07, 3.704030859807972e-08, 4.5552428673545364e-07, 5.504545086409962e-08, 1.532036321805208e-06, 5.083304131403565e-06, 2.1400977345820138e-07, 1.459531858927221e-06, 1.4768386790819932e-05, 1.0635022817950812e-06, 2.9461503800121136e-05, 5.7288212701678276e-05, 6.615699021494947e-06, 7.844223546271678e-06, 0.00015063839964568615, 0.00022636978246737272, 7.973578613018617e-05, 0.00013040869089309126, 0.010868428274989128, 0.988331139087677, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0004166325961705297, 2.250373881906853e-06, 5.27054771737312e-06, 2.7549118385650218e-05, 3.963056951761246e-05, 7.514749540860066e-06, 8.10094024927821e-06, 1.1581340913835447e-05, 1.320597675658064e-05, 6.750728789484128e-05, 2.5212739274138585e-05, 1.486069868406048e-05, 7.521701627410948e-05, 0.00011457462824182585, 0.0002574531245045364, 0.0003666863194666803, 0.0006904753972776234, 0.0014518487732857466, 0.0032460209913551807, 0.0006353403441607952, 0.008745575323700905, 0.015190862119197845, 0.01815020479261875, 0.1330331563949585, 0.8174033164978027, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [1.673920633038506e-05, 8.245287830277448e-08, 3.257373748510872e-08, 2.873728810470766e-08, 1.6217634879467369e-07, 1.5275355425004022e-09, 2.3278557748085404e-08, 5.576718198341268e-09, 1.2825742601307866e-07, 1.3429436762635305e-07, 6.160537857624604e-09, 2.5232478151338e-07, 3.2057448606792605e-06, 2.6277179543399143e-08, 7.619933626301645e-07, 7.901945537014399e-06, 1.3940400833689637e-07, 1.955255868324457e-07, 0.00032022525556385517, 8.173647074727342e-07, 1.562996089887747e-06, 3.763153017644072e-06, 0.0007450500270351768, 0.0037152147851884365, 3.934417327400297e-05, 0.9951443076133728, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00023842290102038532, 5.040124960942194e-06, 1.3208604059400386e-06, 8.599836291978136e-06, 5.166078608453972e-06, 1.3470925352976337e-07, 6.710816364829952e-07, 2.699698029573483e-07, 6.345731435430935e-06, 4.417831223690882e-05, 2.7444934858067427e-07, 2.017119368247222e-05, 1.652665923757013e-05, 8.122791541609331e-07, 0.0001077100241673179, 0.0001239744306076318, 3.5075922824034933e-06, 8.505840924044605e-06, 6.980268517509103e-05, 0.0002681369078345597, 2.8792866942239925e-05, 0.00011957906826864928, 0.0001296169066336006, 0.007425930816680193, 0.0007109709549695253, 0.005815016105771065, 0.9848405718803406, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00040899505256675184, 6.095365279179532e-07, 1.9963911199738504e-06, 9.31032991502434e-06, 3.769046315937885e-06, 1.987656105484348e-06, 4.57654266483587e-07, 1.069074301085493e-06, 3.087174036409124e-06, 9.211397809849586e-06, 4.724369318864774e-06, 1.2070032653355156e-06, 5.095620508654974e-06, 1.8047117919195443e-05, 4.857865133089945e-05, 4.169386738794856e-05, 0.00010537513298913836, 0.00022466867812909186, 0.00011528613686095923, 0.0002492894127499312, 0.0013673142530024052, 0.0016190343303605914, 0.0013275155797600746, 0.003491780487820506, 0.06305663287639618, 0.00646906066685915, 0.0951264277100563, 0.8262877464294434, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00031804729951545596, 1.4181842971083825e-06, 3.1855009297032666e-07, 2.7823105028801365e-06, 5.388588397181593e-05, 4.300602540752152e-08, 5.386270913732005e-07, 8.497841008647811e-08, 5.99971485826245e-07, 4.110329882678343e-06, 7.099399113030813e-08, 8.454308044747449e-07, 2.5266151624236954e-06, 2.198386539475905e-07, 6.098252015362959e-06, 2.1130046661710367e-05, 9.411802466274821e-07, 9.95040522866475e-07, 3.041357376787346e-05, 2.976995119752246e-06, 8.453868758806493e-06, 2.6007013730122708e-05, 0.00023055852216202766, 0.0013383383629843593, 0.0002175826084567234, 0.007014401722699404, 0.017924025654792786, 0.0005302676581777632, 0.9722622632980347, 0.0, 0.0, 0.0, 0.0], [1.8505332377571904e-07, 5.565692351439111e-09, 7.262683787701008e-10, 7.745998509278706e-09, 1.507951097323712e-08, 8.22852175286215e-10, 1.7915348093922034e-09, 1.7345647140842857e-09, 1.0962556196147943e-08, 5.0032749498996054e-08, 3.9228162940219136e-09, 1.0013924622853665e-07, 2.333774595797422e-08, 1.6555254944705666e-08, 5.1713197279923406e-08, 1.614605764643784e-07, 9.827901692460728e-08, 2.707449482386437e-07, 5.411987331171986e-06, 2.5083886612264905e-07, 1.1805956319221877e-06, 2.8004205887555145e-06, 4.61038098364952e-06, 0.00022679210815113038, 5.7942423154599965e-05, 0.0004519550711847842, 0.0001197646779473871, 0.00023980921832844615, 0.001874006469734013, 0.9970145225524902, 0.0, 0.0, 0.0], [4.347462163423188e-05, 2.968874923681142e-06, 3.919368793958711e-07, 9.117065360442211e-07, 4.364489711861097e-07, 4.686591115188321e-09, 9.240378062713717e-08, 1.2419349992853768e-08, 7.188964445958845e-08, 6.304543944679608e-07, 6.108888950251412e-09, 1.9987160726486763e-07, 1.4578674836229766e-06, 1.4727343788933922e-08, 1.7957326292616926e-07, 2.6422712835483253e-05, 5.112656609185251e-08, 4.710674161856332e-08, 0.0015761180547997355, 2.198941047026892e-06, 3.3339856031489035e-07, 2.1814876163261943e-06, 0.00016240392869804054, 0.0017777654575183988, 1.1749207260436378e-05, 0.0036160903982818127, 6.199482595548034e-05, 3.5168231988791376e-05, 0.001148868934251368, 0.016085581853985786, 0.9754422307014465, 0.0, 0.0], [0.0011562927393242717, 6.199047675181646e-06, 6.137794912319805e-07, 3.102660230069887e-06, 8.228408319155278e-07, 4.3660193682626414e-08, 9.36325577072239e-08, 2.7735543639550997e-08, 5.109529865876539e-07, 1.2355309308986762e-06, 3.8253045175906664e-08, 1.127230689235148e-06, 1.9008897425010218e-06, 8.714057031511402e-08, 5.398183020588476e-06, 7.393444320769049e-06, 3.128881189695676e-07, 3.0146313179102435e-07, 0.00013595378550235182, 1.3343835234991275e-05, 2.39964992942987e-06, 3.871273293043487e-05, 0.0008219636511057615, 0.0006131280679255724, 2.151841727027204e-05, 0.004140730947256088, 0.0008760862983763218, 0.00011684626952046528, 0.0021590692922472954, 0.027525268495082855, 0.04434971883893013, 0.9179997444152832, 0.0], [0.003194290678948164, 4.681676728068851e-05, 3.49913134414237e-05, 2.7043113732361235e-05, 1.4000337614561431e-05, 1.7085165382013656e-05, 8.051893928495701e-06, 3.909831775672501e-06, 1.0716959877754562e-05, 1.793081537471153e-05, 1.4207512322172988e-05, 2.691133522603195e-05, 2.6411998987896368e-05, 3.1100229534786195e-05, 1.1198581887583714e-05, 5.485505607794039e-05, 0.00011474490747787058, 0.00010493675654288381, 0.00028063071658834815, 0.00020839076023548841, 0.0009329256135970354, 0.0018990016542375088, 0.0011267587542533875, 0.0019257268868386745, 0.0039171562530100346, 0.007499460596591234, 0.010695938020944595, 0.023199880495667458, 0.03039817325770855, 0.12729351222515106, 0.04325327277183533, 0.1766805499792099, 0.5669293999671936]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.25308629870414734, 0.746913731098175, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.30671578645706177, 0.32906386256217957, 0.3642203211784363, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.07416975498199463, 0.1618966907262802, 0.054325949400663376, 0.7096075415611267, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1668822318315506, 0.039017826318740845, 0.03822460398077965, 0.21398353576660156, 0.5418918132781982, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.19483636319637299, 0.2111925631761551, 0.05150565132498741, 0.0870383232831955, 0.2299954891204834, 0.22543159127235413, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.13900135457515717, 0.02974863536655903, 0.03860695660114288, 0.05133272334933281, 0.19284246861934662, 0.08012373745441437, 0.4683440625667572, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08969778567552567, 0.04080599546432495, 0.03473307564854622, 0.08414525538682938, 0.09911047667264938, 0.07059449702501297, 0.1361657828092575, 0.4447471499443054, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05986519902944565, 0.020190449431538582, 0.018785323947668076, 0.1058472990989685, 0.057948824018239975, 0.027517711743712425, 0.0566631518304348, 0.08226760476827621, 0.5709145069122314, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.010379260405898094, 0.0046982537023723125, 0.004143984522670507, 0.0072914063930511475, 0.006256693042814732, 0.0033180469181388617, 0.009175012819468975, 0.018754413351416588, 0.03393147885799408, 0.9020513892173767, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.07270114123821259, 0.0477648489177227, 0.010500345379114151, 0.0161594245582819, 0.046178195625543594, 0.05249456688761711, 0.051845718175172806, 0.21189512312412262, 0.035158202052116394, 0.17267179489135742, 0.2826306223869324, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.015126891434192657, 0.004731189925223589, 0.0023226316552609205, 0.0065757399424910545, 0.01836245507001877, 0.003339666873216629, 0.008784224279224873, 0.007409745827317238, 0.006289808079600334, 0.07638943940401077, 0.01200348511338234, 0.838664710521698, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.03233193978667259, 0.010628965683281422, 0.0026151640340685844, 0.0011762931244447827, 0.0030932866502553225, 0.0015055524418130517, 0.007079718168824911, 0.00283446884714067, 0.005003884434700012, 0.012032588012516499, 0.0039877742528915405, 0.051872409880161285, 0.8658380508422852, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05045590549707413, 0.02653883770108223, 0.005783271975815296, 0.00808729324489832, 0.021799379959702492, 0.025670334696769714, 0.022779036313295364, 0.09139905869960785, 0.01501737255603075, 0.07093921303749084, 0.12448635697364807, 0.1266833245754242, 0.0610213503241539, 0.34933918714523315, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05474122613668442, 0.05296846479177475, 0.00398812722414732, 0.012351620942354202, 0.00441562058404088, 0.0035962786059826612, 0.011385707184672356, 0.009828065522015095, 0.014749741181731224, 0.07078825682401657, 0.011209187097847462, 0.05164055898785591, 0.1973438560962677, 0.027129262685775757, 0.4738638997077942, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.007163074798882008, 0.004963848739862442, 0.0027692278381437063, 0.0019424431957304478, 0.010544748045504093, 0.0014144877204671502, 0.003663665847852826, 0.003149948548525572, 0.005481299012899399, 0.021614043042063713, 0.003922200761735439, 0.07935082912445068, 0.2247791737318039, 0.009166955016553402, 0.027402665466070175, 0.5926714539527893, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.034611064940690994, 0.015296351164579391, 0.003095126012340188, 0.0038990958128124475, 0.009989401325583458, 0.01144686620682478, 0.009189358912408352, 0.03356517106294632, 0.005761212203651667, 0.025581011548638344, 0.043846867978572845, 0.04521361365914345, 0.02055339701473713, 0.1196233481168747, 0.1567324846982956, 0.13173452019691467, 0.32986119389533997, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.017253872007131577, 0.004397525917738676, 0.004968690220266581, 0.007064312230795622, 0.006634894758462906, 0.006910949479788542, 0.011386900208890438, 0.016178512945771217, 0.020317763090133667, 0.020396802574396133, 0.02342209964990616, 0.01890912838280201, 0.04024204611778259, 0.06200675293803215, 0.116453617811203, 0.07975102961063385, 0.16864468157291412, 0.37506037950515747, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.002459116280078888, 0.00024919764837250113, 3.400466084713116e-05, 0.0002113294176524505, 0.00020066798606421798, 0.0001258924457943067, 0.00036506837932392955, 0.00037359801353886724, 0.00013798549480270594, 0.0004702212172560394, 0.00027151827816851437, 0.003417074913159013, 0.0006049809162504971, 0.0005454609636217356, 0.0008937679813243449, 0.0015852133510634303, 0.0012150758411735296, 0.00282126828096807, 0.9840186238288879, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00015282434469554573, 0.038923632353544235, 0.00047151994658634067, 0.0003731817996595055, 3.271793684689328e-05, 1.063560375769157e-05, 3.8245766518230084e-06, 1.3244694855529815e-05, 6.418924022000283e-05, 2.7344345653546043e-05, 2.8319956982159056e-05, 0.0022295680828392506, 0.0013866383815184236, 7.163518603192642e-05, 0.0001402555499225855, 0.00824644137173891, 0.00019678636454045773, 0.00020546668383758515, 0.8976381421089172, 0.049783725291490555, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.02645810693502426, 0.010410889983177185, 0.0021332555916160345, 0.0024560203310102224, 0.005542597267776728, 0.005954570136964321, 0.0041958331130445, 0.013451533392071724, 0.0023519538808614016, 0.009471958503127098, 0.015433433465659618, 0.014804964885115623, 0.006865760777145624, 0.037873148918151855, 0.047449350357055664, 0.045472823083400726, 0.10199166089296341, 0.08666391670703888, 0.06470184028148651, 0.10160762816667557, 0.3947088122367859, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.021157892420887947, 0.0035484335385262966, 0.001744111767038703, 0.0035093079786747694, 0.004201894626021385, 0.00257777888327837, 0.004015856888145208, 0.004598999861627817, 0.003955156076699495, 0.006455833092331886, 0.00590789457783103, 0.0037331636995077133, 0.009013411588966846, 0.013927980326116085, 0.029895801097154617, 0.02008231170475483, 0.03804440051317215, 0.06883092224597931, 0.0815289318561554, 0.0383402481675148, 0.15160253643989563, 0.48332712054252625, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0014211690286174417, 0.00040690117748454213, 0.0002714421134442091, 0.0014869242440909147, 0.0006475155241787434, 0.0002567152841947973, 0.00027297638007439673, 0.0005059852264821529, 0.00017537492385599762, 0.0012059591244906187, 0.0005902894772589207, 0.00043073901906609535, 0.00036999606527388096, 0.0013322837185114622, 0.000759427435696125, 0.002518307650461793, 0.003599978983402252, 0.004756716545671225, 0.011892932467162609, 0.0034102171193808317, 0.013851108960807323, 0.05907328426837921, 0.8907637596130371, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00505886459723115, 0.0008823543903417885, 0.0015358274104073644, 0.0016909865662455559, 0.0013438736787065864, 0.0004949708236381412, 0.0006572254933416843, 0.000552159093786031, 0.0005812101298943162, 0.002054560463875532, 0.0008380875224247575, 0.001006747712381184, 0.0011300853220745921, 0.0016139923827722669, 0.005022874567657709, 0.014576874673366547, 0.0037143337540328503, 0.004681476857513189, 0.007913822308182716, 0.006778388284146786, 0.012291365303099155, 0.027733024209737778, 0.10214646905660629, 0.7957004308700562, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00400469871237874, 0.0004576975479722023, 0.0015645211096853018, 0.0031372052617371082, 0.001993746031075716, 0.00037892640102654696, 0.00049577810568735, 0.000569801835808903, 0.001005206722766161, 0.0023432248272001743, 0.0005772155127488077, 0.0016756145050749183, 0.003078736364841461, 0.0012285250704735518, 0.005278328433632851, 0.00632805610075593, 0.003151731565594673, 0.0053193108178675175, 0.0213831327855587, 0.006147712003439665, 0.012332223355770111, 0.07402164489030838, 0.1211402490735054, 0.561111569404602, 0.16127507388591766, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0011286542285233736, 4.9889109504874796e-05, 4.129114677198231e-05, 9.24961204873398e-05, 5.513622454600409e-05, 3.8617254176642746e-05, 5.5920278100529686e-05, 0.00014896267384756356, 5.509220500243828e-06, 6.817427492933348e-05, 3.6204455682309344e-05, 4.623961285687983e-05, 0.001065501943230629, 5.91987554798834e-05, 7.787253707647324e-05, 0.00038421223871409893, 0.00011692437692545354, 0.0002242824120912701, 0.000508398690726608, 0.0001562813704367727, 0.00032331488910131156, 0.0007086826371960342, 0.0009207415860146284, 0.0008559715934097767, 0.004414450377225876, 0.9884170889854431, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0024299705401062965, 0.003659637412056327, 0.0006344981375150383, 0.0006202656077221036, 0.00298767676576972, 0.00034763681469485164, 0.0005059166578575969, 0.00041149763274006546, 0.00021321582607924938, 0.0006509345257654786, 0.0003955696302000433, 0.16986453533172607, 0.002188899554312229, 0.0006812380161136389, 0.000868376693688333, 0.008905136026442051, 0.0014638527063652873, 0.0013394575798884034, 0.006495129782706499, 0.007638717070221901, 0.004619399085640907, 0.003161790780723095, 0.014200896956026554, 0.046396613121032715, 0.02278606779873371, 0.21939247846603394, 0.4771406054496765, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.015610319562256336, 0.0009648068808019161, 0.0010281825670972466, 0.007148513570427895, 0.005316650029271841, 0.0005432680482044816, 0.0007839886820875108, 0.0004708298947662115, 0.0022980361245572567, 0.005463092587888241, 0.0005483070272020996, 0.0008661497267894447, 0.0009035550756379962, 0.0009251966839656234, 0.008375934325158596, 0.0014478107914328575, 0.0020614899694919586, 0.003693382255733013, 0.017008859664201736, 0.006138675846159458, 0.007041892036795616, 0.04128709062933922, 0.06922487169504166, 0.26617181301116943, 0.05024212971329689, 0.11445137113332748, 0.13764896988868713, 0.23233483731746674, 0.0, 0.0, 0.0, 0.0, 0.0], [0.021895311772823334, 0.0013514558086171746, 0.0013727976474910975, 0.005157721694558859, 0.01093280129134655, 0.00044522207463160157, 0.0016178301302716136, 0.0005713265272788703, 0.0007700274582020938, 0.003093535080552101, 0.0003616345056798309, 0.0009329011081717908, 0.001903240685351193, 0.0005254389252513647, 0.0007036188035272062, 0.0015780640533193946, 0.0010200472315773368, 0.0009346723672933877, 0.005167578347027302, 0.0007855422445572913, 0.0028796829283237457, 0.006337142549455166, 0.04951414093375206, 0.021886898204684258, 0.03004683181643486, 0.06941598653793335, 0.02995387092232704, 0.061116255819797516, 0.6677284240722656, 0.0, 0.0, 0.0, 0.0], [0.0002946656895801425, 8.08511977083981e-05, 0.00011992631334578618, 0.00017154013039544225, 0.0003899005532730371, 8.630252705188468e-05, 5.253727795206942e-05, 6.616376049350947e-05, 0.0001137837243732065, 6.200082134455442e-05, 0.00011238027218496427, 0.0005867315921932459, 9.785001748241484e-05, 0.00021334877237677574, 0.00040444283513352275, 0.0004726238548755646, 0.0004730736545752734, 0.0005199519800953567, 0.0008571160142309964, 0.00022720196284353733, 0.0015976768918335438, 0.0014780627097934484, 0.0016501408535987139, 0.009339644573628902, 0.01279268879443407, 0.012606951408088207, 0.021659579128026962, 0.02886745147407055, 0.07784783840179443, 0.8267576694488525, 0.0, 0.0, 0.0], [0.0006396545795723796, 0.00019920240447390825, 0.0003378460241947323, 0.00032286165514960885, 0.00038839809712953866, 2.2511459974339232e-05, 0.00011559655831661075, 1.968370270333253e-05, 8.749768312554806e-05, 0.00016675743972882628, 2.1353174815885723e-05, 0.00018257695774082094, 0.002879622159525752, 3.2708376238588244e-05, 6.355983350658789e-05, 0.00090785016072914, 6.31420552963391e-05, 4.9545076763024554e-05, 0.005010190419852734, 0.00021555765124503523, 0.00018923325114883482, 0.00023579830303788185, 0.0021191714331507683, 0.0034752420615404844, 0.0004957506898790598, 0.012673038989305496, 0.0026274591218680143, 0.0011926377192139626, 0.028011513873934746, 0.1421458125114441, 0.7951081991195679, 0.0, 0.0], [0.005492142401635647, 0.0014179605059325695, 6.310038588708267e-05, 0.0009282368700951338, 0.0004675872914958745, 9.701211820356548e-05, 5.678690649801865e-05, 6.824053707532585e-05, 4.054806777276099e-05, 0.00021011428907513618, 6.534699787152931e-05, 0.00011957778770010918, 9.082323231268674e-05, 9.077344293473288e-05, 0.00013827496150042862, 0.00041228361078538, 0.00016900571063160896, 0.00013570708688348532, 0.002428320934996009, 0.00020654806576203555, 0.0004959595971740782, 0.00016561975644435734, 0.001825400977395475, 0.001680815708823502, 0.001018735347315669, 0.01507147029042244, 0.003408371703699231, 0.003384194802492857, 0.01886550709605217, 0.010037437081336975, 0.008259555324912071, 0.923088550567627, 0.0], [0.009386110119521618, 0.0017380556091666222, 0.0018091682577505708, 0.0014350252458825707, 0.0016812816029414535, 0.001979130320250988, 0.0014143437147140503, 0.0021127830259501934, 0.0012016963446512818, 0.0010467646643519402, 0.001253248774446547, 0.0012125695357099175, 0.0006952421390451491, 0.0016850176034495234, 0.001290497719310224, 0.0014949428150430322, 0.003052463522180915, 0.006176403723657131, 0.003993155900388956, 0.002994521288201213, 0.008827597834169865, 0.009542387910187244, 0.010477354750037193, 0.0214514322578907, 0.062213558703660965, 0.026861043646931648, 0.033906422555446625, 0.11987609416246414, 0.05027468875050545, 0.026796849444508553, 0.031165700405836105, 0.10728635638952255, 0.4436681270599365]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10520488768815994, 0.8947951793670654, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.03907204046845436, 0.0020172216463834047, 0.9589106440544128, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.015578641556203365, 0.0008392541203647852, 0.0006979768513701856, 0.9828841090202332, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.008856471627950668, 9.934287845680956e-06, 1.1174832252436318e-05, 0.000302469969028607, 0.9908198714256287, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.35445019602775574, 0.030589278787374496, 0.059889569878578186, 0.022903528064489365, 0.0474759079515934, 0.4846915602684021, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04476895555853844, 0.001566466293297708, 0.00037738497485406697, 0.0002507324970792979, 0.00040889676893129945, 0.0002606561465654522, 0.9523669481277466, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09058346599340439, 0.0011470747413113713, 0.006027346942573786, 0.000546847702935338, 0.0017094232607632875, 0.00378508516587317, 0.002684543142095208, 0.8935161828994751, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0060107228346168995, 2.4289069187943824e-05, 0.00031266745645552874, 1.8682934751268476e-05, 0.0002979248820338398, 9.904525541060138e-06, 6.619561190746026e-06, 7.912206569926639e-07, 0.9933184385299683, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.004546814598143101, 0.0002001934335567057, 0.00029920213273726404, 0.001336393179371953, 0.0003267655265517533, 8.741358215047512e-07, 1.7415688489563763e-05, 2.783414174700738e-07, 0.00029665243346244097, 0.9929754734039307, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.12262558937072754, 0.02392657659947872, 0.03827648609876633, 0.016171958297491074, 0.030366230756044388, 0.29375940561294556, 0.10661352425813675, 0.0705994963645935, 0.04190007597208023, 0.017833102494478226, 0.23792758584022522, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0008121057762764394, 0.00011546660243766382, 0.00013500906061381102, 5.5392923968611285e-05, 0.0001317415590165183, 1.0027196140072192e-06, 1.3799102589473478e-06, 1.6001629887796298e-07, 1.8972989437315846e-06, 5.419660737970844e-07, 4.753100597554294e-07, 0.9987448453903198, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0013333020033314824, 0.00010533027671044692, 0.0007669601473025978, 0.0009021890582516789, 3.5708292216440896e-06, 8.120475740724942e-07, 1.4038076301403635e-07, 9.5612755046659e-08, 1.6865038787727826e-06, 2.9789011023240164e-05, 4.3813363959088747e-07, 1.1965776138822548e-06, 0.9968544840812683, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08159798383712769, 0.021388858556747437, 0.028877155855298042, 0.012675793841481209, 0.02493158169090748, 0.229964941740036, 0.10401230305433273, 0.06472097337245941, 0.03460092470049858, 0.015198714099824429, 0.18564164638519287, 0.020528046414256096, 0.005003852304071188, 0.17085722088813782, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0013488862896338105, 8.92121170181781e-05, 0.000685996375977993, 0.0006963219493627548, 2.1248622942948714e-05, 3.7633941474268795e-07, 6.2267383782455e-07, 4.3072165567537013e-07, 7.091880888765445e-06, 0.0013103694655001163, 1.611375211041377e-07, 9.54876213654643e-07, 0.00016453623538836837, 1.15319252813606e-07, 0.9956737160682678, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0010867505334317684, 0.000407249026466161, 0.00010700321581680328, 0.000678271462675184, 0.00011028484004782513, 1.318426257057581e-07, 2.660271150034532e-07, 4.6195768277357274e-07, 8.409714610024821e-06, 0.0007784898625686765, 5.109449574547398e-08, 8.792079029262823e-07, 4.934294702252373e-05, 3.4893233191723994e-08, 0.000114411988761276, 0.9966580867767334, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05868088826537132, 0.0187857486307621, 0.024067312479019165, 0.011013144627213478, 0.021456381306052208, 0.1911056488752365, 0.09902721643447876, 0.06065867841243744, 0.03016272373497486, 0.014801910147070885, 0.15370330214500427, 0.016615159809589386, 0.004385382868349552, 0.14085331559181213, 0.015014998614788055, 0.0053902058862149715, 0.13427793979644775, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.07323037832975388, 0.02837456949055195, 0.02105824276804924, 0.007611881475895643, 0.01005358062684536, 0.06553948670625687, 0.14559759199619293, 0.08620844036340714, 0.01024126447737217, 0.007425738964229822, 0.045262690633535385, 0.006621456239372492, 0.001162177650257945, 0.03923853859305382, 0.008256880566477776, 0.0030656892340630293, 0.036122631281614304, 0.40492871403694153, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [7.94449370005168e-05, 1.8382123698756914e-06, 3.7134324060161816e-08, 1.5575307770632207e-05, 3.4723757380561437e-06, 1.3165096790501707e-09, 2.703870904952055e-08, 1.7186652101486288e-08, 9.2322629541286e-08, 1.350637921859743e-05, 4.4787887287789374e-10, 2.041673052843862e-09, 5.5420287026208825e-08, 3.0638666603799436e-10, 4.217719435928302e-07, 7.065880708978511e-06, 2.424439260462208e-10, 1.4397358860662735e-09, 0.9998784065246582, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.002934493590146303, 0.0005883841076865792, 0.004250307101756334, 0.0040284618735313416, 0.00011592944065341726, 4.3316217670508195e-06, 6.773899713152787e-06, 7.076467591105029e-05, 0.0002495265216566622, 0.0003159338375553489, 2.08243977795064e-06, 4.4092124880990013e-05, 0.00030446931486949325, 1.4702561657031765e-06, 0.0001976135972654447, 0.0013599261874333024, 1.2478651569836074e-06, 6.875188319099834e-06, 0.00019261146371718496, 0.9853246808052063, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04056532680988312, 0.017898326739668846, 0.020600972697138786, 0.009821100160479546, 0.01788281463086605, 0.15653997659683228, 0.09208228439092636, 0.05628826469182968, 0.026661338284611702, 0.014471272937953472, 0.12537629902362823, 0.013986770063638687, 0.00392792746424675, 0.11453618854284286, 0.013395379297435284, 0.004846664611250162, 0.10880246758460999, 0.049533966928720474, 0.005186409689486027, 0.004420871380716562, 0.10317535698413849, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.02453630231320858, 0.0031243355479091406, 0.0009699289221316576, 4.6315435611177236e-05, 0.0001343475450994447, 0.0006453939131461084, 0.011668838560581207, 0.00032713444670662284, 0.0004573040350805968, 2.3976079319254495e-05, 0.0003445417678449303, 4.996419738745317e-05, 2.5757384719327092e-05, 0.0002821775560732931, 6.525318895000964e-06, 4.981805432180408e-06, 0.00023554006475023925, 0.0002664509229362011, 2.531113386794459e-05, 2.9761044061160646e-05, 0.00018809252651408315, 0.9566071033477783, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.002076001837849617, 0.00014591813669539988, 5.28989803569857e-05, 4.218077810946852e-05, 4.7387235099449754e-05, 4.14233511492057e-07, 0.00012964299821760505, 2.4251021386589855e-05, 1.8034043023362756e-06, 1.4839295090496307e-06, 1.4490007060885546e-07, 4.199936029181117e-06, 4.1068410894240515e-08, 1.0260500005188078e-07, 8.285463763968437e-07, 5.781907930213492e-07, 7.69515864362802e-08, 6.669622365507166e-08, 6.47262713755481e-05, 1.5700916264904663e-06, 5.6418304694716426e-08, 4.403035745781381e-06, 0.997401237487793, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0005159107386134565, 0.00011814631579909474, 3.2705611374694854e-05, 0.00021188720711506903, 4.563520269584842e-05, 2.924884086041857e-07, 3.0579394660890102e-06, 7.607361141026558e-08, 0.0001375367137370631, 0.0005283543141558766, 1.1576013747571778e-07, 4.028577222925378e-06, 2.079294745271909e-06, 8.805408668877135e-08, 0.00010051687422674149, 4.43161916336976e-05, 6.963698950812613e-08, 9.373040654736542e-08, 3.337768794153817e-05, 1.5888888810877688e-05, 5.6106753021367695e-08, 7.05264184830412e-08, 9.389270417159423e-06, 0.9981963038444519, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.06915225088596344, 0.014032105915248394, 0.005599097348749638, 0.0012576853623613715, 0.0008468001033179462, 0.009601173922419548, 0.008965768851339817, 0.052476901561021805, 0.00025028391974046826, 0.0009063858306035399, 0.005419905763119459, 0.000730921165086329, 0.0001918794005177915, 0.004396580625325441, 0.005964720156043768, 0.00035585410660132766, 0.0038770169485360384, 0.009727226570248604, 0.0011023489059880376, 0.0003644288226496428, 0.0034139587078243494, 0.0030523554887622595, 0.005698245484381914, 0.00029557300149463117, 0.7923205494880676, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0010461547644808888, 1.9557141058612615e-05, 1.7358959212288028e-06, 3.1030376703711227e-05, 2.73079531325493e-05, 1.0180698062356441e-08, 4.815767624677392e-07, 1.749424853869641e-07, 7.31240987761339e-08, 7.983068712746899e-07, 3.4643101720632785e-09, 4.9067175496020354e-06, 1.0718932230702194e-07, 2.352543715389288e-09, 2.3834076046114205e-07, 1.3731333865507622e-06, 1.828867168818249e-09, 6.247609984910696e-09, 9.208339179167524e-06, 1.6384256014134735e-06, 1.2523899695082719e-09, 8.271274221094416e-10, 8.456770046905149e-06, 3.118676431768108e-06, 9.241420961814129e-09, 0.9988435506820679, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0010544945253059268, 0.00021373217168729752, 0.00028157219639979303, 0.0011373378802090883, 0.00014783968799747527, 2.00685326490202e-06, 4.831355909118429e-06, 3.48233857039304e-06, 5.100344878883334e-06, 0.000124204860185273, 9.301529075855797e-07, 1.0582622962829191e-05, 1.1688286576827522e-05, 7.077518944242911e-07, 0.003411600599065423, 0.00019905276712961495, 5.995618153065152e-07, 4.643046622732072e-07, 1.9585138943511993e-05, 0.00011087791790487245, 4.7500583377768635e-07, 3.2808881655910227e-07, 4.535358584689675e-06, 5.1617142162285745e-05, 3.247476740853017e-07, 1.3071385183138773e-05, 0.9931889772415161, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.02420075610280037, 0.002820128807798028, 0.004319296684116125, 0.0009108306840062141, 0.0017753936117514968, 0.003809914458543062, 0.0029533228371292353, 0.01671477034687996, 0.001393173704855144, 0.005003183614462614, 0.002276088809594512, 0.0005876408540643752, 0.0009901623707264662, 0.0018066932680085301, 0.0031312420032918453, 0.00023389369016513228, 0.0016082713846117258, 0.00991030503064394, 0.002853952581062913, 0.00020495128410402685, 0.0013712650397792459, 0.00691900635138154, 0.0010336573468521237, 0.00014528384781442583, 0.017489798367023468, 0.00046553125139325857, 0.00030240853084251285, 0.8847692012786865, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0005256092990748584, 3.528650950102019e-06, 1.703768475636025e-06, 9.408179903402925e-05, 0.6647976040840149, 1.6531491553450905e-07, 9.511112466498162e-07, 8.886883620107255e-07, 1.701671317277942e-05, 3.7421868910314515e-05, 6.832266308265389e-08, 9.810898518480826e-06, 7.548737102069936e-08, 4.9632518539510784e-08, 2.0393888462422183e-06, 7.686546268814709e-06, 4.066851388984105e-08, 2.6744315206883584e-08, 3.343187927384861e-05, 4.341078749803273e-07, 3.007988169656528e-08, 3.206056931048806e-08, 4.900574822386261e-06, 3.0609394343628082e-06, 1.4603195985785078e-08, 2.2145539332996123e-05, 9.21115315577481e-06, 3.164272044386962e-08, 0.33442792296409607, 0.0, 0.0, 0.0, 0.0], [0.004677004646509886, 0.0002802134840749204, 8.346093818545341e-05, 3.157015271426644e-06, 4.065350367454812e-07, 1.334351367177078e-07, 2.6423560484545305e-05, 1.14399881567806e-07, 2.787234905099467e-07, 3.7496099594136467e-06, 4.652897445112103e-08, 1.7213982346220291e-06, 1.8165600579322927e-07, 3.1484699292150253e-08, 1.8334249318741058e-07, 5.2147463236451586e-08, 2.5215808108214333e-08, 3.078440968806717e-08, 7.142029545548212e-08, 1.6640171907056356e-08, 1.7768222448921733e-08, 8.634525272555038e-08, 2.80096719507128e-06, 8.454135809188301e-07, 3.098357126418705e-08, 2.927866091795295e-07, 5.67622805647261e-07, 5.0554116448608966e-09, 2.9141034474378102e-08, 0.9949179887771606, 0.0, 0.0, 0.0], [5.636792411678471e-05, 0.00020023711840622127, 1.6319079350068932e-06, 1.5089327689565835e-06, 1.7779733752831817e-05, 5.95644145118257e-10, 6.940369075891795e-06, 5.034019867622419e-09, 2.1682852491267113e-07, 1.7164256860269234e-06, 1.9326265587871205e-10, 1.8911425314627195e-08, 9.538840117784275e-08, 1.2946883842790413e-10, 8.07715050399338e-09, 4.386615546536632e-05, 9.768683778554887e-11, 5.256080015669795e-10, 6.93216934450902e-05, 7.505880716962565e-08, 7.069290985928234e-11, 3.988046870517792e-08, 1.0610276149236597e-05, 2.2120707399153616e-06, 8.688633701403603e-10, 9.928487088473048e-06, 5.390084378831261e-09, 1.1354458345769203e-10, 3.060037442992325e-06, 3.8945552205404965e-07, 0.9995738863945007, 0.0, 0.0], [0.00035569758620113134, 1.9489458281896077e-05, 9.160975423583295e-06, 1.3085252248856705e-05, 2.3808875994291157e-05, 2.97190183573548e-08, 4.1605944716138765e-05, 1.7756508441379992e-07, 6.896241870890663e-07, 1.050624064191652e-06, 1.2230765733534099e-08, 4.290548076824052e-06, 3.295830310889869e-07, 9.21108966878137e-09, 1.7466413737565745e-06, 4.68278949483647e-06, 7.4427397578347154e-09, 9.673819079125678e-09, 3.9665097574470565e-05, 7.324898660954204e-07, 5.5436903956262995e-09, 1.0396706073834139e-07, 1.4401575754163787e-05, 8.873340448189992e-06, 5.98941607421466e-09, 1.5627783795935102e-05, 2.9811219519615406e-06, 6.874342872720263e-09, 4.854650796914939e-06, 7.0722649070376065e-06, 6.210685569385532e-06, 0.999423623085022, 0.0], [0.05455271154642105, 0.01642906479537487, 0.01600833795964718, 0.006986881140619516, 0.02488415502011776, 0.07376932352781296, 0.07933780550956726, 0.05594552680850029, 0.02726820856332779, 0.013368779793381691, 0.05590308830142021, 0.007223027292639017, 0.005426580552011728, 0.05012328550219536, 0.00827906746417284, 0.0030609637033194304, 0.046822965145111084, 0.04751850664615631, 0.0028449445962905884, 0.007037974428385496, 0.042827170342206955, 0.0513819120824337, 0.02728237397968769, 0.026375338435173035, 0.026159211993217468, 0.011997989378869534, 0.006220121867954731, 0.03648780658841133, 0.011124704033136368, 0.005218563135713339, 0.0025592022575438023, 0.0026040268130600452, 0.14697040617465973]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9192656874656677, 0.08073433488607407, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.45437392592430115, 0.4265042543411255, 0.11912186443805695, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5267521739006042, 0.22321631014347076, 0.12420626729726791, 0.12582528591156006, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3903489112854004, 0.17099407315254211, 0.06872710585594177, 0.1837182343006134, 0.1862117052078247, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.19744226336479187, 0.25436851382255554, 0.14488449692726135, 0.18925853073596954, 0.20821937918663025, 0.00582678010687232, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.19473573565483093, 0.13241828978061676, 0.14391927421092987, 0.15120737254619598, 0.3002447485923767, 0.02058463543653488, 0.056889984756708145, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1269666850566864, 0.21597377955913544, 0.16089318692684174, 0.1969882994890213, 0.1884462684392929, 0.013544929213821888, 0.07520709931850433, 0.021979698911309242, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2912123203277588, 0.05954226851463318, 0.11079408973455429, 0.09717857837677002, 0.22345606982707977, 0.016262585297226906, 0.08719441294670105, 0.024762287735939026, 0.08959739655256271, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.22981953620910645, 0.038026757538318634, 0.1601383537054062, 0.11780407279729843, 0.1618119478225708, 0.03552818298339844, 0.07561112195253372, 0.020756039768457413, 0.10670627653598785, 0.05379769951105118, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.13314427435398102, 0.18858656287193298, 0.10808095335960388, 0.15994971990585327, 0.1773921251296997, 0.003598014824092388, 0.0250620748847723, 0.01286292728036642, 0.0745309367775917, 0.11275240778923035, 0.004039982799440622, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1674640029668808, 0.1435522586107254, 0.06118571758270264, 0.17358075082302094, 0.10113830864429474, 0.01755347289144993, 0.03239530324935913, 0.007995500229299068, 0.05453447997570038, 0.07883201539516449, 0.017258215695619583, 0.14450997114181519, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.26996830105781555, 0.13029062747955322, 0.06605685502290726, 0.061561424285173416, 0.08437130600214005, 0.01984662190079689, 0.01631278730928898, 0.012965923175215721, 0.03231464698910713, 0.05254792049527168, 0.01986861415207386, 0.13695642352104187, 0.09693864732980728, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09820353239774704, 0.13618646562099457, 0.0800577849149704, 0.12152864784002304, 0.1357937604188919, 0.0024323570542037487, 0.01778949797153473, 0.009086488746106625, 0.05637527257204056, 0.08521638810634613, 0.0027533371467143297, 0.1520378440618515, 0.0994671955704689, 0.00307140382938087, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1701289564371109, 0.08885712921619415, 0.09089689701795578, 0.08085790276527405, 0.11427336931228638, 0.010901479050517082, 0.014788412488996983, 0.005497370846569538, 0.051512569189071655, 0.05330684036016464, 0.010595815256237984, 0.09442280977964401, 0.13098116219043732, 0.011153682135045528, 0.07182558625936508, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10365856438875198, 0.03588540852069855, 0.08803567290306091, 0.06087937951087952, 0.08348330855369568, 0.03080574795603752, 0.029615936800837517, 0.010918705724179745, 0.04049992188811302, 0.04288605973124504, 0.03219296410679817, 0.15008129179477692, 0.12658776342868805, 0.0349021814763546, 0.05825432389974594, 0.07131282985210419, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.07047097384929657, 0.0960099846124649, 0.056215789169073105, 0.08699887990951538, 0.09934122860431671, 0.0016298787668347359, 0.01240861602127552, 0.006380431819707155, 0.041014038026332855, 0.06348712742328644, 0.0018482195446267724, 0.11387647688388824, 0.07328979671001434, 0.0020712940022349358, 0.07246166467666626, 0.20012779533863068, 0.002367778215557337, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05659440532326698, 0.08906208723783493, 0.04703241214156151, 0.09457876533269882, 0.08793889731168747, 0.001972166122868657, 0.017767680808901787, 0.005990432109683752, 0.06014860421419144, 0.08834964036941528, 0.0022637685760855675, 0.1046152412891388, 0.06079366058111191, 0.00255662202835083, 0.07818624377250671, 0.19496025145053864, 0.0029299911111593246, 0.004259058274328709, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1597062051296234, 0.026096442714333534, 0.04515180364251137, 0.0349758081138134, 0.08030673116445541, 0.05469036474823952, 0.04078545421361923, 0.02651304565370083, 0.02725193277001381, 0.03919816389679909, 0.05351434648036957, 0.0484292209148407, 0.0626426488161087, 0.05558208376169205, 0.05559664964675903, 0.04792959615588188, 0.05676326900720596, 0.04892301186919212, 0.035943206399679184, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08663122355937958, 0.09557436406612396, 0.036738116294145584, 0.06946705281734467, 0.05681459605693817, 0.00819021463394165, 0.022195832803845406, 0.0076428246684372425, 0.03146827220916748, 0.07555203884840012, 0.0089652631431818, 0.05263480916619301, 0.04491106793284416, 0.009935053065419197, 0.04481377825140953, 0.15198466181755066, 0.0109334047883749, 0.007993670180439949, 0.16402564942836761, 0.01352810300886631, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04783454164862633, 0.06669356673955917, 0.03798917308449745, 0.06040368974208832, 0.06899043172597885, 0.0010267288889735937, 0.008172674104571342, 0.004158170893788338, 0.028268268331885338, 0.044072382152080536, 0.0011586467735469341, 0.0783291831612587, 0.05082900822162628, 0.0013004952343180776, 0.049042947590351105, 0.14145004749298096, 0.0014934344217181206, 0.0025566834956407547, 0.17534081637859344, 0.12910208106040955, 0.0017870558658614755, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04114089906215668, 0.04766072705388069, 0.03586998209357262, 0.06666103005409241, 0.07969274371862411, 0.00236507598310709, 0.02034018747508526, 0.004854495171457529, 0.0643620491027832, 0.06737152487039566, 0.0027073686942458153, 0.05596053972840309, 0.05648668855428696, 0.0030767249409109354, 0.05354386568069458, 0.15165923535823822, 0.003539716824889183, 0.0046739280223846436, 0.16402718424797058, 0.05968156456947327, 0.004154939670115709, 0.010169521905481815, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0662880688905716, 0.04092540964484215, 0.02732268162071705, 0.04505367949604988, 0.06575217843055725, 0.015657562762498856, 0.02788187935948372, 0.013711260631680489, 0.03174294903874397, 0.05280746519565582, 0.015447776764631271, 0.032181113958358765, 0.032901059836149216, 0.01629886031150818, 0.0266366358846426, 0.09772849082946777, 0.017114203423261642, 0.01691945269703865, 0.09847065806388855, 0.04787908494472504, 0.018412545323371887, 0.015331599861383438, 0.17753532528877258, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09801430255174637, 0.026328807696700096, 0.03586815297603607, 0.05713058263063431, 0.06943874061107635, 0.012128958478569984, 0.020909087732434273, 0.008747960440814495, 0.02699965611100197, 0.037266768515110016, 0.011716341599822044, 0.08499539643526077, 0.05501333996653557, 0.012196713127195835, 0.03563909977674484, 0.08789661526679993, 0.012970106676220894, 0.010851015336811543, 0.06408385187387466, 0.059639204293489456, 0.01449217926710844, 0.006093734409660101, 0.09761437773704529, 0.053964968770742416, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.02769325114786625, 0.04073771461844444, 0.031223542988300323, 0.058790355920791626, 0.056018248200416565, 0.0015714796027168632, 0.011908041313290596, 0.003339792136102915, 0.051765501499176025, 0.05291244387626648, 0.0018259818898513913, 0.060300350189208984, 0.028357185423374176, 0.0020878082141280174, 0.05086478963494301, 0.07898838073015213, 0.0023987055756151676, 0.002904822351410985, 0.11135734617710114, 0.04586018621921539, 0.0028849171940237284, 0.011431250721216202, 0.1234055683016777, 0.13488474488258362, 0.006487556733191013, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15390776097774506, 0.023786796256899834, 0.05320245772600174, 0.03963885083794594, 0.05711062252521515, 0.015628937631845474, 0.02020607329905033, 0.009252368472516537, 0.02131185494363308, 0.02902994491159916, 0.014741902239620686, 0.0612383633852005, 0.10112591087818146, 0.015157817862927914, 0.04335402697324753, 0.04090719670057297, 0.01573265716433525, 0.020073410123586655, 0.053749360144138336, 0.03468616306781769, 0.01698937825858593, 0.006368495989590883, 0.04042017087340355, 0.031060419976711273, 0.013937168754637241, 0.06738192588090897, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08555091917514801, 0.035057418048381805, 0.05885257571935654, 0.03497908264398575, 0.05178743600845337, 0.0025119921192526817, 0.0041512190364301205, 0.0020859173964709044, 0.014081120491027832, 0.02449135296046734, 0.0023760052863508463, 0.09535116702318192, 0.07766135036945343, 0.0024908811319619417, 0.045726776123046875, 0.05774116516113281, 0.002660380443558097, 0.0033477561082690954, 0.047998279333114624, 0.02678881771862507, 0.0029587154276669025, 0.004655787255614996, 0.03607099875807762, 0.021599192172288895, 0.004788633435964584, 0.22052206099033356, 0.03371305391192436, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.020706716924905777, 0.037194062024354935, 0.01972370594739914, 0.038942236453294754, 0.03182699903845787, 0.0012253678869456053, 0.008815747685730457, 0.002405293518677354, 0.0345795564353466, 0.047298070043325424, 0.001395374652929604, 0.05148206278681755, 0.02048674412071705, 0.0015608376124873757, 0.03826117143034935, 0.04863753169775009, 0.0017825308023020625, 0.0025446040090173483, 0.06541350483894348, 0.0418856181204319, 0.0020960604306310415, 0.009401390329003334, 0.08460027724504471, 0.09019991755485535, 0.0054032751359045506, 0.22932568192481995, 0.05692270025610924, 0.005882916506379843, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08582809567451477, 0.039671752601861954, 0.014841826632618904, 0.056536220014095306, 0.05960613489151001, 0.0069875167682766914, 0.010110098868608475, 0.002887725830078125, 0.015322853811085224, 0.02592875435948372, 0.006830447353422642, 0.033548954874277115, 0.021093523129820824, 0.007213158532977104, 0.019323380663990974, 0.05298228561878204, 0.007658614311367273, 0.005157842300832272, 0.06523183733224869, 0.018170006573200226, 0.008534705266356468, 0.0043681650422513485, 0.026912428438663483, 0.049627941101789474, 0.004404331557452679, 0.19388671219348907, 0.03933224827051163, 0.005363550037145615, 0.11263895034790039, 0.0, 0.0, 0.0, 0.0], [0.07040347903966904, 0.05926162749528885, 0.015334847383201122, 0.0241390373557806, 0.015728840604424477, 0.019492102786898613, 0.017311513423919678, 0.017303066328167915, 0.020563244819641113, 0.0451558418571949, 0.021815825253725052, 0.07079727947711945, 0.030063964426517487, 0.02354060858488083, 0.020684480667114258, 0.02116367779672146, 0.02644512802362442, 0.023540044203400612, 0.05663726478815079, 0.0336947925388813, 0.030707288533449173, 0.015993360430002213, 0.015474079176783562, 0.030803624540567398, 0.03180558606982231, 0.07969609647989273, 0.03437487408518791, 0.034537144005298615, 0.030624371021986008, 0.06290692836046219, 0.0, 0.0, 0.0], [0.05338500067591667, 0.029621528461575508, 0.021030094474554062, 0.023695707321166992, 0.033436402678489685, 0.01839382015168667, 0.024060707539319992, 0.02281002327799797, 0.013972364366054535, 0.028813544660806656, 0.019059743732213974, 0.015371965244412422, 0.02533530630171299, 0.019971953704953194, 0.014075737446546555, 0.0388445146381855, 0.021346209570765495, 0.019321434199810028, 0.056326594203710556, 0.021360008046030998, 0.023394575342535973, 0.023136639967560768, 0.06416381895542145, 0.06724688410758972, 0.0346950888633728, 0.043490175157785416, 0.03032725676894188, 0.03285712003707886, 0.053275078535079956, 0.0659981295466423, 0.041182488203048706, 0.0, 0.0], [0.18177089095115662, 0.019533155485987663, 0.028002941980957985, 0.02911018393933773, 0.04134833440184593, 0.01801948994398117, 0.010471724905073643, 0.008382909931242466, 0.01533578522503376, 0.021839451044797897, 0.01689334586262703, 0.03806138411164284, 0.03643865883350372, 0.017464255914092064, 0.026129068806767464, 0.04011504724621773, 0.018317215144634247, 0.012488218024373055, 0.05136970803141594, 0.01944384165108204, 0.019889770075678825, 0.0035764521453529596, 0.026232963427901268, 0.018429189920425415, 0.011723755858838558, 0.0327298603951931, 0.034808199852705, 0.01203504391014576, 0.06030292063951492, 0.00787330698221922, 0.042272936552762985, 0.07959000021219254, 0.0], [0.015003404580056667, 0.02033485472202301, 0.007813435047864914, 0.016553575173020363, 0.01646609604358673, 0.00023809655976947397, 0.0019598929211497307, 0.0006898260326124728, 0.006483218166977167, 0.010734074749052525, 0.0002443529956508428, 0.014634673483669758, 0.00924762524664402, 0.0002605919726192951, 0.009339917451143265, 0.042726580053567886, 0.0002886394504457712, 0.0003811670758295804, 0.054749373346567154, 0.09452768415212631, 0.00034211305319331586, 0.001727019902318716, 0.04745933413505554, 0.02661379985511303, 0.0010238662362098694, 0.1653798222541809, 0.014187106862664223, 0.0013092352310195565, 0.04770654812455177, 0.0111984983086586, 0.2205076366662979, 0.1394261121749878, 0.0004418626776896417]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9333657622337341, 0.06663428246974945, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.32452359795570374, 0.5923718214035034, 0.08310455828905106, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1415974646806717, 0.514380931854248, 0.28144142031669617, 0.06258013844490051, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.25262272357940674, 0.058340705931186676, 0.10458870232105255, 0.4411159157752991, 0.14333190023899078, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.18051563203334808, 0.09828907996416092, 0.10873956233263016, 0.12262694537639618, 0.09995546191930771, 0.38987332582473755, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1719229817390442, 0.023320995271205902, 0.12332706153392792, 0.06076495721936226, 0.05971711874008179, 0.347591757774353, 0.21335510909557343, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09627032279968262, 0.04683619737625122, 0.028257304802536964, 0.03239249438047409, 0.0586504191160202, 0.15829356014728546, 0.3099648952484131, 0.2693347930908203, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09948235750198364, 0.0646418035030365, 0.028671829029917717, 0.0730566680431366, 0.02978716976940632, 0.1396002322435379, 0.20754705369472504, 0.265802264213562, 0.09141054004430771, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05632037669420242, 0.029885854572057724, 0.01341991126537323, 0.0024075533729046583, 0.013245469890534878, 0.045645419508218765, 0.05522569268941879, 0.10027101635932922, 0.6579986810684204, 0.02558007836341858, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04266994819045067, 0.010426479391753674, 0.010652213357388973, 0.013446621596813202, 0.00935314130038023, 0.04015978425741196, 0.0710517093539238, 0.1373765617609024, 0.10423263162374496, 0.21736638247966766, 0.34326446056365967, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0296839140355587, 0.002561642089858651, 0.004407483618706465, 0.034248705953359604, 0.014238743111491203, 0.028001822531223297, 0.02228802628815174, 0.07105964422225952, 0.04026233032345772, 0.5543041825294495, 0.17333054542541504, 0.025612974539399147, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04039592295885086, 0.008631582371890545, 0.005494222976267338, 0.003864138852804899, 0.023363104090094566, 0.027207026258111, 0.025677336379885674, 0.050485219806432724, 0.0510561428964138, 0.05041782185435295, 0.14164337515830994, 0.511528491973877, 0.060235634446144104, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.028706390410661697, 0.004808460362255573, 0.004723656922578812, 0.005382601171731949, 0.003456039587035775, 0.014470163732767105, 0.024037377908825874, 0.041626207530498505, 0.03567296266555786, 0.07060738652944565, 0.10830992460250854, 0.05811614543199539, 0.2062121331691742, 0.3938705325126648, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05342644825577736, 0.011923279613256454, 0.015572259202599525, 0.0021123222541064024, 0.005851038731634617, 0.02217092178761959, 0.03818469122052193, 0.02741156704723835, 0.054642871022224426, 0.04910830780863762, 0.12328378856182098, 0.05287173017859459, 0.07545649260282516, 0.3701530694961548, 0.09783118963241577, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.008346711285412312, 0.00027852741186507046, 0.0011785993119701743, 0.0021395485382527113, 0.002476169029250741, 0.002918292535468936, 0.004576893523335457, 0.0039655971340835094, 0.004973022732883692, 0.011347771622240543, 0.013014075346291065, 0.005545974709093571, 0.02089560590684414, 0.03838539123535156, 0.8750316500663757, 0.004926194902509451, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.02170032449066639, 0.002678664866834879, 0.0023839243222028017, 0.002815627260133624, 0.001556233735755086, 0.006139412056654692, 0.009039806202054024, 0.01374087855219841, 0.012412214651703835, 0.02468654327094555, 0.03406905382871628, 0.017561480402946472, 0.06082957610487938, 0.12004052847623825, 0.11516803503036499, 0.12663689255714417, 0.42854079604148865, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.01968139037489891, 0.0028661887627094984, 0.002752861240878701, 0.0023642012383788824, 0.0022205759305506945, 0.0047769746743142605, 0.005015391390770674, 0.007020775228738785, 0.006603003013879061, 0.01582488976418972, 0.02177913300693035, 0.02237590402364731, 0.03905554488301277, 0.07342638820409775, 0.07227564603090286, 0.0810362920165062, 0.2556319534778595, 0.36529290676116943, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.034438684582710266, 0.0024295076727867126, 0.0027038780972361565, 0.0010108931455761194, 0.0028212429024279118, 0.009310661815106869, 0.00747407041490078, 0.010526392608880997, 0.01934259757399559, 0.006583388429135084, 0.030525274574756622, 0.008166542276740074, 0.023945646360516548, 0.08047989010810852, 0.04428938403725624, 0.048865143209695816, 0.23266319930553436, 0.38709720969200134, 0.04732644557952881, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0036976831033825874, 0.006450952962040901, 0.0004921727231703699, 0.0003069589729420841, 0.00012400773994158953, 0.0010727370390668511, 0.0008858796209096909, 0.001380636473186314, 0.0008200175943784416, 0.0002731183485593647, 0.002783065428957343, 0.03529426082968712, 0.0035613791551440954, 0.007148314733058214, 0.001647408353164792, 0.016731660813093185, 0.01917281188070774, 0.022339196875691414, 0.8733668327331543, 0.002450960921123624, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.013628787361085415, 0.001438445644453168, 0.0012188641121611, 0.0012915851548314095, 0.0006118023884482682, 0.0021400381810963154, 0.0027469571214169264, 0.0033360477536916733, 0.003091872902587056, 0.005988912656903267, 0.007237219717353582, 0.003547744592651725, 0.012891951948404312, 0.022229159250855446, 0.020824221894145012, 0.02629532851278782, 0.0764816403388977, 0.16261443495750427, 0.1263866275548935, 0.10034442692995071, 0.4056538939476013, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.021432969719171524, 0.0007291649235412478, 0.0010889691766351461, 0.0008559620473533869, 0.0010556603083387017, 0.002354251453652978, 0.0018633343279361725, 0.002486992860212922, 0.004011375363916159, 0.0036456233356148005, 0.006636620499193668, 0.0024591651745140553, 0.00810664240270853, 0.01950138807296753, 0.013945515267550945, 0.01683160476386547, 0.06727856397628784, 0.1020374447107315, 0.07506179064512253, 0.03785506263375282, 0.3474159240722656, 0.2633460462093353, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.01837441883981228, 0.0016340153524652123, 0.0002648660447448492, 0.0006277099018916488, 0.0007212890777736902, 0.0022947092074900866, 0.0014375777682289481, 0.0024335708003491163, 0.0010225051082670689, 0.002173094777390361, 0.005665970034897327, 0.000585706380661577, 0.004730311222374439, 0.01578962802886963, 0.009121065959334373, 0.006447889842092991, 0.049651872366666794, 0.07221098989248276, 0.04530426487326622, 0.02253621071577072, 0.23380489647388458, 0.28100958466529846, 0.22215785086154938, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.01601027511060238, 0.00041907597915269434, 0.0016793111572042108, 0.0020331626292318106, 0.0011126026511192322, 0.0023041211534291506, 0.001641488284803927, 0.0020452297758311033, 0.0008826447883620858, 0.003531813621520996, 0.00466074887663126, 0.001146126538515091, 0.004025280009955168, 0.011729470454156399, 0.006906012073159218, 0.005978981498628855, 0.03413612022995949, 0.053919851779937744, 0.07536919414997101, 0.015544144436717033, 0.153072789311409, 0.21672524511814117, 0.23267339169979095, 0.1524529755115509, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.01641550101339817, 0.0010881004855036736, 0.001302703283727169, 0.001233565155416727, 0.0010325239272788167, 0.0013605316635221243, 0.001135275000706315, 0.0009713207837194204, 0.0017580230487510562, 0.002354995347559452, 0.0022677984088659286, 0.0026170071214437485, 0.0032849612180143595, 0.00556444376707077, 0.0072877430357038975, 0.006582668051123619, 0.0169111005961895, 0.023348137736320496, 0.033363793045282364, 0.01206086203455925, 0.08410514146089554, 0.10776982456445694, 0.12741318345069885, 0.2500958740711212, 0.2886749505996704, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.031117741018533707, 0.0015063054161146283, 0.0010839324677363038, 0.002421524142846465, 0.001382686896249652, 0.0037937266752123833, 0.001633929554373026, 0.003119518980383873, 0.0012812635395675898, 0.002068523084744811, 0.0047601195983588696, 0.0008757320465520024, 0.008592123165726662, 0.0093536376953125, 0.011993243359029293, 0.005963001865893602, 0.022028174251317978, 0.03411327302455902, 0.009586759842932224, 0.013681814074516296, 0.08417054265737534, 0.08917812258005142, 0.10806837677955627, 0.15560784935951233, 0.37236863374710083, 0.020249491557478905, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.014838481321930885, 0.0005757318576797843, 0.004743653815239668, 0.00045677347225137055, 0.0007412288105115294, 0.0015000010607764125, 0.0008866073912940919, 0.0008452099282294512, 0.00010359263978898525, 0.000906514236703515, 0.0018096797866746783, 0.0602911040186882, 0.001537756179459393, 0.0033079395070672035, 0.0013713664375245571, 0.002082675462588668, 0.008643986657261848, 0.009088865481317043, 0.007760955020785332, 0.00580833712592721, 0.033865127712488174, 0.03987184166908264, 0.05374552309513092, 0.036947719752788544, 0.12259436398744583, 0.5718704462051392, 0.013804465532302856, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.01857001520693302, 0.0012143177445977926, 0.001288044499233365, 0.0010612568585202098, 0.0007490046555176377, 0.001048562815412879, 0.000868787697982043, 0.0005973864463157952, 0.0012938270810991526, 0.001404251204803586, 0.0011055096983909607, 0.0005003446131013334, 0.0020262624602764845, 0.0021463141310960054, 0.0028737925458699465, 0.0026413712184876204, 0.005802977364510298, 0.0081913648173213, 0.00833908375352621, 0.005543305538594723, 0.026558682322502136, 0.03308920934796333, 0.032507359981536865, 0.09199848771095276, 0.10410483181476593, 0.16861778497695923, 0.208954319357872, 0.2669035494327545, 0.0, 0.0, 0.0, 0.0, 0.0], [0.016654442995786667, 0.0007248665788210928, 0.0008980666170828044, 0.0031616878695786, 0.0005522824940271676, 0.0016607288271188736, 0.00074861227767542, 0.0009652797016315162, 0.0003842804580926895, 0.0022577813360840082, 0.0015545616624876857, 0.0007037006434984505, 0.0014688181690871716, 0.002799882087856531, 0.003013978712260723, 0.005369038321077824, 0.006552346982061863, 0.008701568469405174, 0.007138868793845177, 0.006516328547149897, 0.026185575872659683, 0.03929203003644943, 0.03155006840825081, 0.02434305101633072, 0.1067076250910759, 0.15159794688224792, 0.13674212992191315, 0.3201395273208618, 0.09161490201950073, 0.0, 0.0, 0.0, 0.0], [0.0372268483042717, 0.0007442747591994703, 0.0019180340459570289, 0.0014290640829131007, 0.0025597368367016315, 0.003292836481705308, 0.0009007275220938027, 0.001544511178508401, 0.0003557214804459363, 0.0004492733278311789, 0.0026688158977776766, 0.0014046661090105772, 0.0017281017499044538, 0.004542736802250147, 0.0012941033346578479, 0.004677668213844299, 0.009138188324868679, 0.008992945775389671, 0.005343662574887276, 0.0020318389870226383, 0.030930904671549797, 0.021215269342064857, 0.03639978542923927, 0.056794628500938416, 0.11195950210094452, 0.011458109132945538, 0.03353285789489746, 0.2658853828907013, 0.2469542920589447, 0.09262555837631226, 0.0, 0.0, 0.0], [0.044088784605264664, 0.0011082949349656701, 0.0033875906374305487, 0.004853191319853067, 0.002864320995286107, 0.004500862676650286, 0.0014518116367980838, 0.0021924942266196012, 0.0007902359939180315, 0.0006717380601912737, 0.003577157622203231, 0.0005131773068569601, 0.012861719354987144, 0.0051168459467589855, 0.0030142527539283037, 0.005887466017156839, 0.010453412309288979, 0.013818331062793732, 0.008691059425473213, 0.011229269206523895, 0.0314009003341198, 0.023181650787591934, 0.03044436313211918, 0.03520682081580162, 0.09999866038560867, 0.006090165581554174, 0.14714214205741882, 0.16393053531646729, 0.12749530375003815, 0.1935775727033615, 0.0004599907260853797, 0.0, 0.0], [0.03717399016022682, 0.005006894003599882, 0.0016112301964312792, 0.0028301470447331667, 0.0005118494154885411, 0.0027071270160377026, 0.0020970311015844345, 0.0012328773736953735, 0.0006310884491540492, 0.001763244392350316, 0.001786878565326333, 0.00025283300783485174, 0.0010720754507929087, 0.0026448338758200407, 0.0015661576762795448, 0.0038084082771092653, 0.005276741925626993, 0.006930254865437746, 0.01331083569675684, 0.0035816438030451536, 0.018304521217942238, 0.026684748008847237, 0.019971538335084915, 0.03215576708316803, 0.07432040572166443, 0.023922255262732506, 0.08761478215456009, 0.1800757199525833, 0.027492573484778404, 0.059722453355789185, 0.19122688472270966, 0.1627122461795807, 0.0], [0.00811693910509348, 0.0008723069913685322, 0.0003707293944898993, 0.00047348643420264125, 0.0001783298939699307, 0.0003994710568804294, 0.00029511720640584826, 0.00018311945314053446, 0.000167145932209678, 0.00023702479666098952, 0.0002533360675442964, 0.00012855060049332678, 0.00020213205425534397, 0.0003994489670731127, 0.0001651181373745203, 0.00046228704741224647, 0.0008780225180089474, 0.0011140474816784263, 0.0020708786323666573, 0.0019636773504316807, 0.0035278573632240295, 0.0044752187095582485, 0.004463486839085817, 0.007312532514333725, 0.014446244575083256, 0.035030465573072433, 0.02001415751874447, 0.04388578608632088, 0.019436758011579514, 0.11325737833976746, 0.07277627289295197, 0.16196827590465546, 0.4804745018482208]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.947374701499939, 0.05262523517012596, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8785883784294128, 0.055394671857357025, 0.06601690500974655, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7270770072937012, 0.1370987594127655, 0.04866093024611473, 0.08716330677270889, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4651746451854706, 0.13429315388202667, 0.1535816341638565, 0.1747254729270935, 0.07222513109445572, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1703597903251648, 0.011841950938105583, 0.0236508846282959, 0.021745815873146057, 0.025856325402855873, 0.7465452551841736, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14214572310447693, 0.04383273795247078, 0.050211820751428604, 0.04516506567597389, 0.058721259236335754, 0.5111533999443054, 0.1487700343132019, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0847334936261177, 0.020369820296764374, 0.03746161609888077, 0.03195912018418312, 0.03419598937034607, 0.43500685691833496, 0.1189405769109726, 0.2373325079679489, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3423079550266266, 0.03593634441494942, 0.0725921243429184, 0.10352761298418045, 0.08196527510881424, 0.07694562524557114, 0.14196324348449707, 0.09896710515022278, 0.045794595032930374, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3711485266685486, 0.06278738379478455, 0.05806950107216835, 0.055662769824266434, 0.13124839961528778, 0.07535629719495773, 0.10726629197597504, 0.07924024015665054, 0.030107343569397926, 0.029113225638866425, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.06430108100175858, 0.0051691061817109585, 0.01101299561560154, 0.009054888971149921, 0.011748063378036022, 0.3188501000404358, 0.03684947267174721, 0.0831955149769783, 0.012068143114447594, 0.014676210470497608, 0.433074414730072, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.23777922987937927, 0.03812997415661812, 0.01974336989223957, 0.07508028298616409, 0.14420795440673828, 0.06244930252432823, 0.07018385827541351, 0.06523600965738297, 0.048832181841135025, 0.03460032865405083, 0.06439767777919769, 0.13935980200767517, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.29014477133750916, 0.04085766524076462, 0.06965630501508713, 0.06467737257480621, 0.11515594273805618, 0.07279439270496368, 0.05406038090586662, 0.040959976613521576, 0.036461226642131805, 0.03709695488214493, 0.08112155646085739, 0.06249447911977768, 0.03451891988515854, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04191606491804123, 0.0033880723640322685, 0.007082367315888405, 0.005899207200855017, 0.007586182560771704, 0.20389790832996368, 0.023296764120459557, 0.05173072963953018, 0.007816805504262447, 0.00973137654364109, 0.2776019871234894, 0.008386771194636822, 0.01617271639406681, 0.3354930877685547, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15129348635673523, 0.04240793362259865, 0.043833814561367035, 0.09706760942935944, 0.14943556487560272, 0.044398821890354156, 0.0617450475692749, 0.04199514910578728, 0.018608780577778816, 0.031412459909915924, 0.05092642828822136, 0.06660290062427521, 0.09416216611862183, 0.057612884789705276, 0.04849693179130554, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.29958024621009827, 0.07432066649198532, 0.03006197139620781, 0.016736475750803947, 0.038564808666706085, 0.05106877535581589, 0.09408668428659439, 0.05606791749596596, 0.054582275450229645, 0.045220401138067245, 0.056380707770586014, 0.015378053300082684, 0.04082852229475975, 0.06084694340825081, 0.030811049044132233, 0.03546451777219772, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.030439963564276695, 0.0024933936074376106, 0.005298204720020294, 0.004223393276333809, 0.005517445504665375, 0.14508457481861115, 0.01603415422141552, 0.03598647937178612, 0.0057580480352044106, 0.007398170419037342, 0.19613467156887054, 0.006090890150517225, 0.011465118266642094, 0.23737992346286774, 0.008031771518290043, 0.005861491896212101, 0.2768023610115051, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.015668945387005806, 0.0019133484456688166, 0.006747933570295572, 0.005610871594399214, 0.006710356567054987, 0.10858529061079025, 0.013436974957585335, 0.030674796551465988, 0.00869804061949253, 0.005958769004791975, 0.15696409344673157, 0.00640126271173358, 0.01435719896107912, 0.19220466911792755, 0.009938806295394897, 0.008154557086527348, 0.22861403226852417, 0.1793600618839264, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2751207649707794, 0.046356070786714554, 0.025782747194170952, 0.02037108689546585, 0.04519129917025566, 0.047415051609277725, 0.05800836905837059, 0.054494574666023254, 0.02185583859682083, 0.028674187138676643, 0.05054797977209091, 0.03578946739435196, 0.014343999326229095, 0.05505189672112465, 0.019970891997218132, 0.0250706784427166, 0.05983670800924301, 0.10591268539428711, 0.01020571868866682, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.12210460752248764, 0.029006367549300194, 0.02792610600590706, 0.030763691291213036, 0.06431955099105835, 0.05075869709253311, 0.05746712535619736, 0.03854454681277275, 0.02881445176899433, 0.03641033545136452, 0.05530688911676407, 0.02755570411682129, 0.05062380060553551, 0.06039686128497124, 0.03790910169482231, 0.034162603318691254, 0.06550325453281403, 0.08224605768918991, 0.08903136104345322, 0.011148839257657528, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.01992272026836872, 0.0015991260297596455, 0.003536272095516324, 0.0027606096118688583, 0.0036747483536601067, 0.09198574721813202, 0.009868193417787552, 0.021762730553746223, 0.0036746191326528788, 0.004800298251211643, 0.12327182292938232, 0.003948224242776632, 0.007727704476565123, 0.14901942014694214, 0.005094495136290789, 0.0038495466578751802, 0.17433811724185944, 0.1450374871492386, 0.003139176871627569, 0.0039015670772641897, 0.21708732843399048, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.025169141590595245, 0.0025891433469951153, 0.007904480211436749, 0.002818175358697772, 0.008317488245666027, 0.06284766644239426, 0.016097376123070717, 0.024833567440509796, 0.007658041547983885, 0.011317074298858643, 0.08451006561517715, 0.012135406024754047, 0.022048376500606537, 0.1001725122332573, 0.015103276818990707, 0.020152254030108452, 0.11906814575195312, 0.141489639878273, 0.006576653569936752, 0.007074056193232536, 0.1485065221786499, 0.1536109447479248, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08679644763469696, 0.01581868715584278, 0.009913153015077114, 0.005941125564277172, 0.01174909994006157, 0.04402247071266174, 0.0494115874171257, 0.047211576253175735, 0.022448517382144928, 0.02814415469765663, 0.05249372497200966, 0.013664884492754936, 0.0380445197224617, 0.05856490507721901, 0.01600872538983822, 0.028531765565276146, 0.06752399355173111, 0.1000405102968216, 0.026074493303894997, 0.01816527359187603, 0.08061447739601135, 0.14588385820388794, 0.03293200954794884, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.12203323096036911, 0.02679692953824997, 0.03471272066235542, 0.016600849106907845, 0.05098596587777138, 0.033623479306697845, 0.03376290947198868, 0.03289441019296646, 0.022763589397072792, 0.03469543159008026, 0.03710823133587837, 0.02750304900109768, 0.030508041381835938, 0.04070492461323738, 0.019409826025366783, 0.033388134092092514, 0.046037882566452026, 0.08141940832138062, 0.025792937725782394, 0.017713353037834167, 0.05326378345489502, 0.06955216079950333, 0.07340100407600403, 0.03532780334353447, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.012632706202566624, 0.003196730511263013, 0.00899962242692709, 0.003787106368690729, 0.0055902148596942425, 0.060187432914972305, 0.010173025541007519, 0.026063401252031326, 0.008280332200229168, 0.005998437758535147, 0.08480245620012283, 0.007755732629448175, 0.014807655476033688, 0.10079342871904373, 0.011636020615696907, 0.007885102182626724, 0.120613694190979, 0.12819863855838776, 0.006487111561000347, 0.00323057291097939, 0.15047506988048553, 0.10557688027620316, 0.015480604022741318, 0.01902633160352707, 0.0783216655254364, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.11176053434610367, 0.018635563552379608, 0.05622374638915062, 0.046772800385951996, 0.03736491501331329, 0.02299342304468155, 0.031365279108285904, 0.026408886536955833, 0.043359220027923584, 0.056635573506355286, 0.021786397323012352, 0.023822149261832237, 0.030641673132777214, 0.023289257660508156, 0.05430489778518677, 0.06881187111139297, 0.024489382281899452, 0.04777085781097412, 0.036271754652261734, 0.022222526371479034, 0.026998935267329216, 0.02176610752940178, 0.03442488983273506, 0.028240257874131203, 0.05891920626163483, 0.024719828739762306, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10639805346727371, 0.0371079258620739, 0.08273956179618835, 0.01785685308277607, 0.020949997007846832, 0.018305253237485886, 0.02262011356651783, 0.018192270770668983, 0.021851180121302605, 0.01251852884888649, 0.019214097410440445, 0.03034009225666523, 0.0718761533498764, 0.020787999033927917, 0.03963525965809822, 0.06151590868830681, 0.02317928522825241, 0.0439804382622242, 0.04619227349758148, 0.02066798508167267, 0.027080876752734184, 0.04386550188064575, 0.0327674001455307, 0.04369068145751953, 0.05080847069621086, 0.04029490798711777, 0.025563018396496773, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.015084554441273212, 0.003822384402155876, 0.005877268500626087, 0.0049718520604074, 0.005370147991925478, 0.05223645642399788, 0.016127675771713257, 0.03182988613843918, 0.00821112934499979, 0.011473809368908405, 0.06704685091972351, 0.010528381913900375, 0.010598219931125641, 0.07938898354768753, 0.007211714517325163, 0.0062714628875255585, 0.09142373502254486, 0.10866505652666092, 0.003973816987127066, 0.005621673539280891, 0.1098332554101944, 0.09237755835056305, 0.02296048402786255, 0.019314926117658615, 0.08864143490791321, 0.018464695662260056, 0.011022249236702919, 0.0916503295302391, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09854131937026978, 0.03967057913541794, 0.05052143707871437, 0.04794390872120857, 0.0195318553596735, 0.015683406963944435, 0.02507476694881916, 0.014997915364801884, 0.02772345580160618, 0.016418185085058212, 0.016389017924666405, 0.026316562667489052, 0.03435927629470825, 0.017757989466190338, 0.0233626626431942, 0.03763514757156372, 0.019405605271458626, 0.034421682357788086, 0.04902297630906105, 0.030190544202923775, 0.022159066051244736, 0.037540629506111145, 0.038793135434389114, 0.04032415151596069, 0.0423126220703125, 0.06829195469617844, 0.025314129889011383, 0.04330073297023773, 0.03699527308344841, 0.0, 0.0, 0.0, 0.0], [0.10688814520835876, 0.03659575805068016, 0.005083094350993633, 0.01016042660921812, 0.013297352008521557, 0.04175207018852234, 0.0262468159198761, 0.03012855537235737, 0.013482386246323586, 0.018223188817501068, 0.0458262674510479, 0.02555879019200802, 0.015618832781910896, 0.05039895698428154, 0.01165701076388359, 0.011659128591418266, 0.0551551915705204, 0.0509166419506073, 0.019206784665584564, 0.013998407870531082, 0.06383473426103592, 0.02908892184495926, 0.016710441559553146, 0.02609526738524437, 0.05484986677765846, 0.03806670755147934, 0.03896071016788483, 0.05375232174992561, 0.023679165169596672, 0.0531080923974514, 0.0, 0.0, 0.0], [0.10871604084968567, 0.06286415457725525, 0.02304510399699211, 0.012493208050727844, 0.009688262827694416, 0.02372078038752079, 0.025617338716983795, 0.028282862156629562, 0.019523099064826965, 0.015187123790383339, 0.024615760892629623, 0.018855812028050423, 0.013071928173303604, 0.02569747343659401, 0.010639789514243603, 0.007336440030485392, 0.028488244861364365, 0.040875114500522614, 0.023693937808275223, 0.02846083790063858, 0.031781155616045, 0.03399210050702095, 0.01690451242029667, 0.04699751362204552, 0.054809220135211945, 0.0604817196726799, 0.021137842908501625, 0.05147771164774895, 0.01530434750020504, 0.07868867367506027, 0.03755194693803787, 0.0, 0.0], [0.16510623693466187, 0.013307320885360241, 0.022820336744189262, 0.032224975526332855, 0.028327813372015953, 0.011912115849554539, 0.021759122610092163, 0.012591322883963585, 0.025371331721544266, 0.01592349074780941, 0.011745063588023186, 0.021340155974030495, 0.022658517584204674, 0.012096998281776905, 0.05877390503883362, 0.041142962872982025, 0.012866133823990822, 0.02189669944345951, 0.02738575078547001, 0.015412718988955021, 0.014822259545326233, 0.013198777101933956, 0.029060106724500656, 0.04031305015087128, 0.023036926984786987, 0.05035790055990219, 0.046500690281391144, 0.03221071884036064, 0.04574136063456535, 0.011448168195784092, 0.05148483067750931, 0.04716223478317261, 0.0], [0.019862741231918335, 0.0006941381143406034, 0.0017670642118901014, 0.0019038202008232474, 0.0023115782532840967, 0.05924388766288757, 0.005164953414350748, 0.008714546449482441, 0.0024906685575842857, 0.0030579573940485716, 0.07585158944129944, 0.002390549285337329, 0.0031773503869771957, 0.08945121616125107, 0.0030652948189526796, 0.0016209169989451766, 0.10312031954526901, 0.0625915676355362, 0.002058295765891671, 0.002232219558209181, 0.12742049992084503, 0.024703850969672203, 0.0035101925022900105, 0.00499417819082737, 0.028317121788859367, 0.005874201189726591, 0.006215325556695461, 0.03686564415693283, 0.005438121035695076, 0.004061868414282799, 0.004735523369163275, 0.004534382838755846, 0.2925584614276886]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9726406335830688, 0.027359364554286003, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8850566744804382, 0.08160645514726639, 0.03333686292171478, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7844979166984558, 0.09045999497175217, 0.10720972716808319, 0.01783234067261219, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6679065227508545, 0.10135099291801453, 0.10997921228408813, 0.0699087604880333, 0.050854478031396866, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.586279034614563, 0.06960930675268173, 0.06798022985458374, 0.06058962643146515, 0.08339767158031464, 0.13214409351348877, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5015710592269897, 0.06227877363562584, 0.07209988683462143, 0.05590919405221939, 0.08146880567073822, 0.12033165246248245, 0.10634062439203262, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.40016087889671326, 0.05525398626923561, 0.06347683072090149, 0.05413786694407463, 0.07119617611169815, 0.11230527609586716, 0.12489721924066544, 0.11857166141271591, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4018532931804657, 0.08579416573047638, 0.056448839604854584, 0.07181122153997421, 0.08318912237882614, 0.07845646142959595, 0.09191881120204926, 0.07768876850605011, 0.05283927917480469, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3827159106731415, 0.07976946234703064, 0.062077559530735016, 0.06360235810279846, 0.08211023360490799, 0.07804954797029495, 0.08597437292337418, 0.07506486773490906, 0.0632784441113472, 0.02735721692442894, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3404167592525482, 0.04535197094082832, 0.04658910632133484, 0.0403917022049427, 0.05489042401313782, 0.08179006725549698, 0.08906516432762146, 0.09645897150039673, 0.05564969405531883, 0.06398677080869675, 0.08540938794612885, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.31186187267303467, 0.09539996832609177, 0.07292013615369797, 0.0697687417268753, 0.07777827978134155, 0.06735038757324219, 0.06161406263709068, 0.054810743778944016, 0.052520137280225754, 0.05022520199418068, 0.07299348711967468, 0.012756960466504097, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.36652863025665283, 0.06308672577142715, 0.03947845473885536, 0.04609525203704834, 0.05740750953555107, 0.0634833425283432, 0.05238465964794159, 0.05501618608832359, 0.04584015905857086, 0.06228693574666977, 0.0658990740776062, 0.058063607662916183, 0.024429509416222572, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.28564438223838806, 0.03804025053977966, 0.03942390903830528, 0.034238625317811966, 0.0463939793407917, 0.06771662831306458, 0.07328074425458908, 0.07855433225631714, 0.047896627336740494, 0.055936526507139206, 0.07124947756528854, 0.035689257085323334, 0.04956042021512985, 0.07637479156255722, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.297754168510437, 0.05259547755122185, 0.04347941279411316, 0.05845862627029419, 0.04568730294704437, 0.05259405076503754, 0.04456199333071709, 0.0459747239947319, 0.06316147744655609, 0.0776875764131546, 0.05499143525958061, 0.03466471657156944, 0.05020103231072426, 0.05886140838265419, 0.01932653971016407, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.28853145241737366, 0.04616911709308624, 0.028233066201210022, 0.038242388516664505, 0.04330621287226677, 0.05174916982650757, 0.04318275675177574, 0.043931201100349426, 0.05616341903805733, 0.06773821264505386, 0.05490177869796753, 0.08139309287071228, 0.04325690492987633, 0.0590481236577034, 0.03989904746413231, 0.014254068955779076, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.23721373081207275, 0.03161581605672836, 0.03324565663933754, 0.029231945052742958, 0.038618117570877075, 0.05549168214201927, 0.05966496840119362, 0.06321781128644943, 0.040736980736255646, 0.04752674326300621, 0.058378204703330994, 0.03039967454969883, 0.04287158325314522, 0.06284616142511368, 0.053212910890579224, 0.04736898094415665, 0.06835903972387314, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2098696380853653, 0.02729126438498497, 0.031156614422798157, 0.026615004986524582, 0.03165113925933838, 0.05273226648569107, 0.052745021879673004, 0.05635902285575867, 0.04076546058058739, 0.04278237372636795, 0.056191232055425644, 0.030678272247314453, 0.04398961737751961, 0.06091473996639252, 0.0501369833946228, 0.0415731742978096, 0.06652864068746567, 0.07801953703165054, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2385382503271103, 0.04414580762386322, 0.046435050666332245, 0.0345127172768116, 0.0357217937707901, 0.04369329661130905, 0.04304974898695946, 0.034988246858119965, 0.043718066066503525, 0.0395219661295414, 0.044578537344932556, 0.024599451571702957, 0.0625118762254715, 0.04753483459353447, 0.043756093829870224, 0.04995930567383766, 0.050908081233501434, 0.0526624359190464, 0.019164469093084335, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.17729151248931885, 0.042214617133140564, 0.030818725004792213, 0.030886096879839897, 0.033868495374917984, 0.045211877673864365, 0.042901039123535156, 0.032215580344200134, 0.040188658982515335, 0.049268968403339386, 0.04801624268293381, 0.03814242035150528, 0.051265135407447815, 0.05206841975450516, 0.047441162168979645, 0.03924533724784851, 0.05630478635430336, 0.05462942644953728, 0.0599186010658741, 0.02810293436050415, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.19048276543617249, 0.025203485041856766, 0.026787471026182175, 0.02367268316447735, 0.03110402822494507, 0.04345344379544258, 0.04649331048130989, 0.048206478357315063, 0.03323280066251755, 0.03892753645777702, 0.04549149051308632, 0.02444273605942726, 0.035701580345630646, 0.04907767102122307, 0.04303564876317978, 0.039055656641721725, 0.0537174753844738, 0.06658957898616791, 0.04320826008915901, 0.03176470100879669, 0.06035127118229866, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14975358545780182, 0.023474570363759995, 0.025508398190140724, 0.028942441567778587, 0.03517158329486847, 0.038801901042461395, 0.04594280198216438, 0.03962729126214981, 0.037087779492139816, 0.038723014295101166, 0.04190719500184059, 0.021688053384423256, 0.04328255355358124, 0.04592541232705116, 0.03893951326608658, 0.0400761142373085, 0.05060023441910744, 0.060830533504486084, 0.043721601366996765, 0.03023521602153778, 0.05728611722588539, 0.062474027276039124, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.17513407766819, 0.027519281953573227, 0.029514014720916748, 0.02071532793343067, 0.026873527094721794, 0.0345461331307888, 0.038530781865119934, 0.033037811517715454, 0.04181556776165962, 0.04010028764605522, 0.03696895390748978, 0.030468937009572983, 0.039953943341970444, 0.04023702070116997, 0.0448262095451355, 0.03708262741565704, 0.04420144855976105, 0.04931037127971649, 0.04125744849443436, 0.028610678389668465, 0.04997344687581062, 0.05269285663962364, 0.036629173904657364, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.13285990059375763, 0.040708061307668686, 0.04111678898334503, 0.034844279289245605, 0.04087604954838753, 0.027567215263843536, 0.028293700888752937, 0.022520417347550392, 0.0387541837990284, 0.040378037840127945, 0.029089409857988358, 0.029609227553009987, 0.0567011833190918, 0.031181858852505684, 0.037661172449588776, 0.056886378675699234, 0.03396708890795708, 0.037813056260347366, 0.04951409250497818, 0.031945571303367615, 0.037599027156829834, 0.04326637089252472, 0.043056681752204895, 0.033790234476327896, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14709368348121643, 0.018218709155917168, 0.022143620997667313, 0.02162139303982258, 0.02707289345562458, 0.03385418280959129, 0.03787611797451973, 0.03801965340971947, 0.027932504191994667, 0.03090183064341545, 0.03606922924518585, 0.01902313157916069, 0.03465801104903221, 0.03916986659169197, 0.033755578100681305, 0.02613663300871849, 0.042978327721357346, 0.05313742905855179, 0.03463505581021309, 0.02014029398560524, 0.04878358915448189, 0.04990803450345993, 0.04624392092227936, 0.04715671017765999, 0.06346960365772247, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1760278195142746, 0.01488215196877718, 0.026751182973384857, 0.0240880586206913, 0.03566748648881912, 0.027094293385744095, 0.02498425543308258, 0.021471885964274406, 0.026359373703598976, 0.023845119401812553, 0.027566982433199883, 0.01686953380703926, 0.0777742937207222, 0.029109211638569832, 0.04799698293209076, 0.02900250256061554, 0.03132336959242821, 0.03615599498152733, 0.03959561511874199, 0.017008014023303986, 0.034839075058698654, 0.03443719446659088, 0.04149218648672104, 0.09124380350112915, 0.040522992610931396, 0.0038906026165932417, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1850174218416214, 0.027787016704678535, 0.025097640231251717, 0.027018999680876732, 0.029199207201600075, 0.030337024480104446, 0.032221775501966476, 0.02301507070660591, 0.0267928596585989, 0.022793851792812347, 0.030674852430820465, 0.021220838651061058, 0.053888075053691864, 0.03259299322962761, 0.030737651512026787, 0.025725245475769043, 0.03486606478691101, 0.037935830652713776, 0.028672009706497192, 0.026148982346057892, 0.03875335678458214, 0.04179506003856659, 0.03863157331943512, 0.05337152257561684, 0.03982071578502655, 0.02571161463856697, 0.010172729380428791, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.12446325272321701, 0.01676889881491661, 0.02010510489344597, 0.019267024472355843, 0.024457814171910286, 0.03020336478948593, 0.03127393499016762, 0.03254607319831848, 0.023186003789305687, 0.02350839227437973, 0.0317331999540329, 0.017080510035157204, 0.029082054272294044, 0.03428216651082039, 0.03579127788543701, 0.025820568203926086, 0.037531185895204544, 0.04618249833583832, 0.029500003904104233, 0.017913807183504105, 0.04274731129407883, 0.042434703558683395, 0.039002396166324615, 0.04141838476061821, 0.05949503183364868, 0.03595735505223274, 0.030984872952103615, 0.057262834161520004, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14491021633148193, 0.025843990966677666, 0.031176971271634102, 0.01946275867521763, 0.012575727887451649, 0.025274591520428658, 0.022020690143108368, 0.021267279982566833, 0.03165539726614952, 0.02766416408121586, 0.026251589879393578, 0.022091038525104523, 0.03851794824004173, 0.028266867622733116, 0.0288776233792305, 0.036865364760160446, 0.03069997765123844, 0.03204502537846565, 0.03411627188324928, 0.02035541459918022, 0.03446315601468086, 0.03542865440249443, 0.030728410929441452, 0.04441893845796585, 0.03793596476316452, 0.05734865367412567, 0.03889700025320053, 0.04111205041408539, 0.019728200510144234, 0.0, 0.0, 0.0, 0.0], [0.13168425858020782, 0.025460105389356613, 0.02796240895986557, 0.024109845981001854, 0.01686880923807621, 0.031406790018081665, 0.023247336968779564, 0.022071687504649162, 0.027362508699297905, 0.02637045457959175, 0.032045334577560425, 0.02877580001950264, 0.04614202678203583, 0.03455977141857147, 0.03132109344005585, 0.02943773753941059, 0.03749070689082146, 0.03597421571612358, 0.0306710172444582, 0.026778314262628555, 0.041662365198135376, 0.03274398297071457, 0.030348768457770348, 0.032651983201503754, 0.038056500256061554, 0.034575607627630234, 0.022936122491955757, 0.03690328821539879, 0.02596619911491871, 0.01441489439457655, 0.0, 0.0, 0.0], [0.1656370609998703, 0.02280924655497074, 0.029107755050063133, 0.02494465932250023, 0.02435287833213806, 0.02503182739019394, 0.018486661836504936, 0.01857367716729641, 0.028443630784749985, 0.029199941083788872, 0.0253182053565979, 0.03220498561859131, 0.025753282010555267, 0.026890520006418228, 0.031273938715457916, 0.03305148705840111, 0.028835458680987358, 0.029910407960414886, 0.0275588259100914, 0.024624649435281754, 0.0320536270737648, 0.02649376355111599, 0.02170960046350956, 0.031640153378248215, 0.03707476332783699, 0.0479947030544281, 0.02793215773999691, 0.034756116569042206, 0.03804944083094597, 0.02153567597270012, 0.008750851266086102, 0.0, 0.0], [0.16794215142726898, 0.025983478873968124, 0.03222338482737541, 0.019469385966658592, 0.023735277354717255, 0.023115742951631546, 0.0199736300855875, 0.016550231724977493, 0.029796775430440903, 0.028494667261838913, 0.023108843713998795, 0.02208707109093666, 0.042165566235780716, 0.02439018338918686, 0.02074914425611496, 0.038735877722501755, 0.026269812136888504, 0.02728910744190216, 0.019617389887571335, 0.020242862403392792, 0.02920321375131607, 0.030413366854190826, 0.02576262876391411, 0.040849827229976654, 0.033326659351587296, 0.021795300766825676, 0.026310516521334648, 0.03101349249482155, 0.03689264506101608, 0.01937715709209442, 0.04355606809258461, 0.009558655321598053, 0.0], [0.11396828293800354, 0.01775287464261055, 0.01606196165084839, 0.015585055574774742, 0.01839612051844597, 0.024367082864046097, 0.026053881272673607, 0.023999109864234924, 0.021919487044215202, 0.026251113042235374, 0.024730533361434937, 0.015323017723858356, 0.025614570826292038, 0.0263378843665123, 0.024982867762446404, 0.02464262954890728, 0.028590328991413116, 0.03295150771737099, 0.027568619698286057, 0.01848754845559597, 0.03220851346850395, 0.031914692372083664, 0.036307260394096375, 0.0341540202498436, 0.036785583943128586, 0.03297772631049156, 0.026278911158442497, 0.03809891268610954, 0.029801947996020317, 0.018183261156082153, 0.03632068261504173, 0.035680945962667465, 0.05770310387015343]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8058414459228516, 0.19415856897830963, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6373865008354187, 0.09672051668167114, 0.26589304208755493, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5870265364646912, 0.09922721236944199, 0.07913831621408463, 0.23460792005062103, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4194737374782562, 0.06312593817710876, 0.08123067021369934, 0.08381500095129013, 0.35235461592674255, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5308531522750854, 0.0959072858095169, 0.07779339700937271, 0.06661548465490341, 0.07293295115232468, 0.15589767694473267, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.38503071665763855, 0.07478094846010208, 0.06978150457143784, 0.047606538981199265, 0.051449161022901535, 0.08776707202196121, 0.2835841178894043, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2962956726551056, 0.05328270420432091, 0.06944749504327774, 0.04097547382116318, 0.06660831719636917, 0.1021856963634491, 0.10869503766298294, 0.26250961422920227, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.30474990606307983, 0.06337985396385193, 0.05753423646092415, 0.04057719185948372, 0.0730014219880104, 0.07375827431678772, 0.07239888608455658, 0.05693604797124863, 0.25766414403915405, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.29880762100219727, 0.05552824214100838, 0.05443597584962845, 0.05285302922129631, 0.06568750739097595, 0.07554222643375397, 0.06968103349208832, 0.054113879799842834, 0.09120790660381317, 0.18214264512062073, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.26933014392852783, 0.06352431327104568, 0.059521228075027466, 0.052424702793359756, 0.06052157282829285, 0.12185832113027573, 0.07608654350042343, 0.09753072261810303, 0.04232607036828995, 0.05058327317237854, 0.10629311949014664, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.22074301540851593, 0.06960077583789825, 0.048723481595516205, 0.06675717234611511, 0.06954211741685867, 0.06483786553144455, 0.046523742377758026, 0.039300836622714996, 0.018016092479228973, 0.0358063168823719, 0.053022876381874084, 0.26712578535079956, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15080073475837708, 0.04732475057244301, 0.12551115453243256, 0.06240082159638405, 0.05354631692171097, 0.05578393489122391, 0.04542611539363861, 0.03888600319623947, 0.0301998108625412, 0.033370546996593475, 0.04645953327417374, 0.023437919095158577, 0.2868523597717285, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2044883519411087, 0.04972049966454506, 0.04838546738028526, 0.043853580951690674, 0.05115717276930809, 0.1007489338517189, 0.06420683115720749, 0.08302561193704605, 0.03880453109741211, 0.047957729548215866, 0.09582382440567017, 0.03239700198173523, 0.04441104456782341, 0.0950193703174591, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1623673439025879, 0.028519174084067345, 0.047843094915151596, 0.041354816406965256, 0.040284883230924606, 0.0520053468644619, 0.04226505383849144, 0.03821168839931488, 0.05986882373690605, 0.06849399209022522, 0.04734278842806816, 0.019715160131454468, 0.06202228367328644, 0.04572192579507828, 0.24398362636566162, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.13128933310508728, 0.039628103375434875, 0.059519413858652115, 0.03966323286294937, 0.04406355693936348, 0.04348496347665787, 0.03726547583937645, 0.03240034729242325, 0.03720574453473091, 0.06481601297855377, 0.040474534034729004, 0.025914235040545464, 0.06507598608732224, 0.03922511264681816, 0.07744196802377701, 0.2225320190191269, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15813341736793518, 0.03911251947283745, 0.03856496512889862, 0.03587968647480011, 0.04187794774770737, 0.08161000907421112, 0.05245806276798248, 0.0681268721818924, 0.03444807231426239, 0.04342752322554588, 0.08275996893644333, 0.02933340333402157, 0.041573796421289444, 0.08478686958551407, 0.04881599545478821, 0.032352838665246964, 0.08673813939094543, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.13890595734119415, 0.03169632703065872, 0.03251556307077408, 0.03538898378610611, 0.038000334054231644, 0.06974425166845322, 0.059424545615911484, 0.06768239289522171, 0.027843810617923737, 0.039054665714502335, 0.07273104041814804, 0.030109861865639687, 0.04163092002272606, 0.07565011829137802, 0.040188319981098175, 0.027483616024255753, 0.07795906811952591, 0.09399020671844482, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09691290557384491, 0.030687319114804268, 0.030817966908216476, 0.04473433643579483, 0.04171944037079811, 0.036036767065525055, 0.04101703688502312, 0.032104767858982086, 0.04344063252210617, 0.048855531960725784, 0.03457901254296303, 0.016025088727474213, 0.049941662698984146, 0.0346352756023407, 0.060656867921352386, 0.034977883100509644, 0.03454342484474182, 0.037827931344509125, 0.25048619508743286, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.12059216946363449, 0.04121065139770508, 0.0509851798415184, 0.052891526371240616, 0.02886129543185234, 0.04706199839711189, 0.0292128287255764, 0.038170017302036285, 0.021046487614512444, 0.029787806794047356, 0.04406721517443657, 0.02650333382189274, 0.04127003997564316, 0.04430737346410751, 0.03080155700445175, 0.04064903408288956, 0.044745106250047684, 0.042122457176446915, 0.02844228595495224, 0.1972716897726059, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.11388304829597473, 0.028255397453904152, 0.028351297602057457, 0.026958664879202843, 0.031310852617025375, 0.05977548286318779, 0.0391821525990963, 0.05072040110826492, 0.027684014290571213, 0.035564228892326355, 0.06452134251594543, 0.02426709607243538, 0.03616495057940483, 0.06838168203830719, 0.04139300435781479, 0.029094640165567398, 0.0723004937171936, 0.07349380850791931, 0.037000205367803574, 0.0352863185107708, 0.07641094923019409, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08453849703073502, 0.017416605725884438, 0.01894914172589779, 0.0165606327354908, 0.02229611948132515, 0.04174060747027397, 0.05894864723086357, 0.03893362358212471, 0.030214650556445122, 0.034642331302165985, 0.0483190156519413, 0.016358422115445137, 0.02567455545067787, 0.05309673771262169, 0.02054651454091072, 0.02046983130276203, 0.057046402245759964, 0.06604909896850586, 0.03774644806981087, 0.021413879469037056, 0.06129930540919304, 0.2077389657497406, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1124843955039978, 0.024394847452640533, 0.024523155763745308, 0.027489561587572098, 0.026104671880602837, 0.03731505200266838, 0.06748531758785248, 0.03890375792980194, 0.01798143796622753, 0.028454387560486794, 0.03756532073020935, 0.01832606829702854, 0.020088663324713707, 0.03865170478820801, 0.0232856422662735, 0.024226617068052292, 0.03960612043738365, 0.04833188280463219, 0.04500093311071396, 0.017362147569656372, 0.04009140655398369, 0.05208287015557289, 0.19024400413036346, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08183357119560242, 0.019828449934720993, 0.02687339298427105, 0.029252557083964348, 0.02518443763256073, 0.03192773088812828, 0.029049793258309364, 0.024972520768642426, 0.03350626677274704, 0.04007429629564285, 0.033517319709062576, 0.016035988926887512, 0.03204454854130745, 0.03448064625263214, 0.028517093509435654, 0.03532731533050537, 0.03583228215575218, 0.04058396443724632, 0.029941106215119362, 0.02109791524708271, 0.03671329841017723, 0.039212584495544434, 0.03543241322040558, 0.23876062035560608, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10145692527294159, 0.014237047173082829, 0.016984183341264725, 0.018636571243405342, 0.02853623777627945, 0.0365380235016346, 0.02957051247358322, 0.041983190923929214, 0.014166045933961868, 0.030081408098340034, 0.04016641527414322, 0.02007502317428589, 0.030370114371180534, 0.043534401804208755, 0.030393019318580627, 0.020717334002256393, 0.04677111282944679, 0.0549510233104229, 0.028809182345867157, 0.020043687894940376, 0.05092543363571167, 0.04084467142820358, 0.0418580137193203, 0.03848976269364357, 0.15986064076423645, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.07937924563884735, 0.020674310624599457, 0.020799757912755013, 0.027988232672214508, 0.04565475508570671, 0.02656259387731552, 0.02539670839905739, 0.021243257448077202, 0.01776379905641079, 0.02520064264535904, 0.027576463297009468, 0.030607089400291443, 0.034380000084638596, 0.028568314388394356, 0.022726934403181076, 0.01605195552110672, 0.029712749645113945, 0.03276180848479271, 0.025702400133013725, 0.012878884561359882, 0.030626127496361732, 0.02036965638399124, 0.03741392120718956, 0.027850378304719925, 0.035718515515327454, 0.27639153599739075, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09605434536933899, 0.02309289574623108, 0.024199659004807472, 0.024974144995212555, 0.036829639226198196, 0.03001439943909645, 0.021069226786494255, 0.018117526546120644, 0.022569267079234123, 0.02834322676062584, 0.030505143105983734, 0.021707680076360703, 0.028211787343025208, 0.031639426946640015, 0.03916612267494202, 0.03627365827560425, 0.03311571851372719, 0.03514396771788597, 0.02574523724615574, 0.015889015048742294, 0.034675464034080505, 0.02677008882164955, 0.027143454179167747, 0.0404508002102375, 0.037603847682476044, 0.032323356717824936, 0.17837099730968475, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.07476937770843506, 0.011021512560546398, 0.010706555098295212, 0.015325457789003849, 0.01990356110036373, 0.03285866603255272, 0.021936072036623955, 0.02945413999259472, 0.01642150804400444, 0.022476578131318092, 0.03722343221306801, 0.01758023351430893, 0.021954145282506943, 0.040462132543325424, 0.031415410339832306, 0.010417776182293892, 0.044297393411397934, 0.05118127167224884, 0.029565559700131416, 0.015394334681332111, 0.0490642674267292, 0.03202195093035698, 0.030905120074748993, 0.04132053256034851, 0.06578273326158524, 0.03978404775261879, 0.02049970254302025, 0.16625656187534332, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05124205723404884, 0.011390028521418571, 0.02026580646634102, 0.024529647082090378, 0.12732437252998352, 0.020083913579583168, 0.017692744731903076, 0.020381992682814598, 0.018548619002103806, 0.023206066340208054, 0.021026061847805977, 0.020521236583590508, 0.025210734456777573, 0.021994242444634438, 0.017190473154187202, 0.02080591395497322, 0.023035308346152306, 0.026458054780960083, 0.031086130067706108, 0.011454472318291664, 0.024357995018363, 0.02252495475113392, 0.024600783362984657, 0.024588001891970634, 0.02838081493973732, 0.04660612717270851, 0.043627094477415085, 0.02030046284198761, 0.21156589686870575, 0.0, 0.0, 0.0, 0.0], [0.07929293811321259, 0.023537395521998405, 0.022847289219498634, 0.01993604749441147, 0.015462876297533512, 0.025110295042395592, 0.021429050713777542, 0.015073496848344803, 0.00992409698665142, 0.015724744647741318, 0.027188707143068314, 0.020747287198901176, 0.03497635945677757, 0.029102325439453125, 0.021219033747911453, 0.01814761571586132, 0.03105819784104824, 0.03380950167775154, 0.020137565210461617, 0.012323119677603245, 0.0336926244199276, 0.02433612570166588, 0.031084517017006874, 0.03377701714634895, 0.04355444014072418, 0.032963525503873825, 0.03314930200576782, 0.02536490187048912, 0.022163692861795425, 0.2228659689426422, 0.0, 0.0, 0.0], [0.10059655457735062, 0.03096376359462738, 0.025578510016202927, 0.0202240701764822, 0.027132991701364517, 0.0294907595962286, 0.0407448373734951, 0.021403610706329346, 0.016392609104514122, 0.02252069301903248, 0.027555176988244057, 0.013599206693470478, 0.027306707575917244, 0.027806581929326057, 0.011214177124202251, 0.023586539551615715, 0.02839917503297329, 0.03066891059279442, 0.021664204075932503, 0.017032310366630554, 0.02905159257352352, 0.026549918577075005, 0.03297552466392517, 0.026790324598550797, 0.032060641795396805, 0.02729833871126175, 0.017307184636592865, 0.018768811598420143, 0.028226524591445923, 0.017085110768675804, 0.18000467121601105, 0.0, 0.0], [0.08361606299877167, 0.016581889241933823, 0.015823302790522575, 0.022005166858434677, 0.02406255342066288, 0.021028300747275352, 0.022811293601989746, 0.014412387274205685, 0.01557208877056837, 0.015104830265045166, 0.020917734131217003, 0.021798150613904, 0.02598598226904869, 0.02201320417225361, 0.022025059908628464, 0.018091218546032906, 0.02311510592699051, 0.025451799854636192, 0.02594352327287197, 0.011241885833442211, 0.024466173723340034, 0.0228698942810297, 0.031613439321517944, 0.029542673379182816, 0.02381790056824684, 0.029349016025662422, 0.025922643020749092, 0.017268173396587372, 0.0318446047604084, 0.012037734501063824, 0.027524925768375397, 0.2561413049697876, 0.0], [0.05258440971374512, 0.012508383020758629, 0.012737675569951534, 0.012144193053245544, 0.0144678158685565, 0.024812448769807816, 0.01710914634168148, 0.02516256831586361, 0.016298966482281685, 0.015160080045461655, 0.029820043593645096, 0.014290276914834976, 0.024094797670841217, 0.033742766827344894, 0.021632254123687744, 0.018924105912446976, 0.03819216787815094, 0.04115181416273117, 0.024165764451026917, 0.020635638386011124, 0.044185325503349304, 0.03793709725141525, 0.029636353254318237, 0.03152839466929436, 0.04542306438088417, 0.035963598638772964, 0.030482539907097816, 0.04609914869070053, 0.03206766024231911, 0.01766294240951538, 0.03169601783156395, 0.04462910816073418, 0.10305341333150864]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7707713842391968, 0.22922858595848083, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7813623547554016, 0.109785296022892, 0.10885230451822281, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.535038948059082, 0.11194943636655807, 0.15013694763183594, 0.20287470519542694, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.41803574562072754, 0.1603638231754303, 0.1074029952287674, 0.10625781118869781, 0.20793959498405457, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4349866509437561, 0.09579858183860779, 0.08251801878213882, 0.10028504580259323, 0.15332908928394318, 0.1330825835466385, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.44621357321739197, 0.05624214559793472, 0.05040299892425537, 0.07084295898675919, 0.09360906481742859, 0.11472620815038681, 0.16796305775642395, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3609195351600647, 0.07294464111328125, 0.05191005766391754, 0.07159046083688736, 0.09321668744087219, 0.09543665498495102, 0.14539918303489685, 0.10858277231454849, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3171406686306, 0.07683130353689194, 0.05552283674478531, 0.06445366144180298, 0.08586590737104416, 0.10643255710601807, 0.1452113687992096, 0.09439337253570557, 0.05414833128452301, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.19288603961467743, 0.1134386956691742, 0.04530505836009979, 0.08361203223466873, 0.05763273686170578, 0.11914196610450745, 0.11441227793693542, 0.12364513427019119, 0.07640958577394485, 0.07351639866828918, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.25174087285995483, 0.060194116085767746, 0.04930208995938301, 0.059223245829343796, 0.08453336358070374, 0.07307681441307068, 0.12308554351329803, 0.08293969184160233, 0.06380272656679153, 0.07185835391283035, 0.08024313300848007, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2479359656572342, 0.048557743430137634, 0.061716966331005096, 0.06616783887147903, 0.0878860130906105, 0.05920383334159851, 0.09128588438034058, 0.06866813451051712, 0.060628652572631836, 0.08399352431297302, 0.0656941756606102, 0.05826116353273392, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15039603412151337, 0.0707230269908905, 0.036188021302223206, 0.03812884911894798, 0.05496574938297272, 0.09552797675132751, 0.1278020143508911, 0.09101750701665878, 0.04133947193622589, 0.06619426608085632, 0.11493087559938431, 0.03526817262172699, 0.0775180533528328, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.21730750799179077, 0.052248772233724594, 0.04279686510562897, 0.05130988731980324, 0.07021071761846542, 0.06084864214062691, 0.10094168037176132, 0.06729137152433395, 0.05347270891070366, 0.05803239718079567, 0.063866026699543, 0.0414368212223053, 0.05429811403155327, 0.06593842804431915, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14158985018730164, 0.08735613524913788, 0.041096266359090805, 0.07771317660808563, 0.0664159283041954, 0.06084612384438515, 0.07890869677066803, 0.06467185169458389, 0.041567541658878326, 0.05314023420214653, 0.060331959277391434, 0.049903471022844315, 0.06147686019539833, 0.06152055785059929, 0.053461313247680664, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.11277392506599426, 0.05140271782875061, 0.03480074182152748, 0.044109929352998734, 0.04927106201648712, 0.08066023141145706, 0.09371130168437958, 0.07634277641773224, 0.03460073471069336, 0.0555649995803833, 0.09000370651483536, 0.027950923889875412, 0.058086883276700974, 0.09759874641895294, 0.058530546724796295, 0.03459078073501587, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.19136789441108704, 0.04588800296187401, 0.037184182554483414, 0.04478847235441208, 0.058455415070056915, 0.0516231432557106, 0.08451797068119049, 0.056617289781570435, 0.04477924853563309, 0.04681586101651192, 0.05177522450685501, 0.034743115305900574, 0.04365590214729309, 0.05257752537727356, 0.045719120651483536, 0.05523938685655594, 0.05425224080681801, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.178098663687706, 0.03651316463947296, 0.035378020256757736, 0.04660060256719589, 0.056067463010549545, 0.05542533099651337, 0.06580902636051178, 0.058695923537015915, 0.04460207000374794, 0.03845333680510521, 0.05522071197628975, 0.02812928520143032, 0.038876280188560486, 0.056352097541093826, 0.04307202994823456, 0.048592109233140945, 0.05801301822066307, 0.056100811809301376, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.11277126520872116, 0.04416341334581375, 0.01679084822535515, 0.03922899439930916, 0.030570238828659058, 0.08429381251335144, 0.05829833820462227, 0.07985574752092361, 0.027740228921175003, 0.024607932195067406, 0.0856802761554718, 0.01213553361594677, 0.021223612129688263, 0.08979952335357666, 0.02439562790095806, 0.018000086769461632, 0.0979786366224289, 0.08564859628677368, 0.04681731015443802, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15648671984672546, 0.043593376874923706, 0.029469136148691177, 0.02338278852403164, 0.03989394009113312, 0.06601448357105255, 0.07239893823862076, 0.049598775804042816, 0.025720087811350822, 0.028729889541864395, 0.07268749177455902, 0.020104380324482918, 0.038949914276599884, 0.0779348835349083, 0.031850170344114304, 0.029572613537311554, 0.08162925392389297, 0.04983029142022133, 0.04757083207368851, 0.014582027681171894, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.16675154864788055, 0.040216755121946335, 0.031946271657943726, 0.03915555030107498, 0.04867977276444435, 0.043519362807273865, 0.07027734071016312, 0.04676978662610054, 0.03737045079469681, 0.03785951808094978, 0.04185349866747856, 0.029438791796565056, 0.03496702387928963, 0.04187152162194252, 0.036659594625234604, 0.04418949410319328, 0.04244700446724892, 0.04308999702334404, 0.055408261716365814, 0.02402997761964798, 0.04349849000573158, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.21311703324317932, 0.03452147915959358, 0.026512037962675095, 0.037284668534994125, 0.04200230911374092, 0.05378075689077377, 0.06410719454288483, 0.040195900946855545, 0.029513953253626823, 0.030562065541744232, 0.048140499740839005, 0.019645733758807182, 0.023855963721871376, 0.04712359979748726, 0.027805231511592865, 0.025632821023464203, 0.04690323397517204, 0.042114999145269394, 0.04245338588953018, 0.016209973022341728, 0.04720157012343407, 0.041315577924251556, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.16099101305007935, 0.03149211406707764, 0.025505999103188515, 0.03203628212213516, 0.038119371980428696, 0.05386463925242424, 0.05289200320839882, 0.06297367811203003, 0.03872942551970482, 0.03070300817489624, 0.04861500486731529, 0.016964146867394447, 0.020984871312975883, 0.04743487387895584, 0.026178555563092232, 0.032817039638757706, 0.048741020262241364, 0.03702303394675255, 0.05028446391224861, 0.014425938948988914, 0.04926920309662819, 0.02748514525592327, 0.052469104528427124, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.13924191892147064, 0.04732196778059006, 0.022920068353414536, 0.037826746702194214, 0.03733159974217415, 0.056281980127096176, 0.06773107498884201, 0.0638374611735344, 0.028824560344219208, 0.028368612751364708, 0.05008407309651375, 0.014775178395211697, 0.021058905869722366, 0.04819713160395622, 0.017685124650597572, 0.01545622292906046, 0.0484280101954937, 0.037099532783031464, 0.042473260313272476, 0.01842682436108589, 0.04861331358551979, 0.022673960775136948, 0.05998954549431801, 0.02535295858979225, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.195623517036438, 0.03233327716588974, 0.022874081507325172, 0.038390763103961945, 0.03449878469109535, 0.04616905376315117, 0.055537495762109756, 0.03960668295621872, 0.02309839427471161, 0.024249285459518433, 0.04043305665254593, 0.016078554093837738, 0.021624945104122162, 0.03903055936098099, 0.026520881801843643, 0.028113437816500664, 0.03823632746934891, 0.035426102578639984, 0.04174575209617615, 0.02109713852405548, 0.03767955303192139, 0.037286918610334396, 0.04423920437693596, 0.022921495139598846, 0.037184685468673706, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0581965334713459, 0.038897398859262466, 0.02600524015724659, 0.02356790192425251, 0.022979989647865295, 0.053841520100831985, 0.058061059564352036, 0.06858771294355392, 0.037459827959537506, 0.029159093275666237, 0.05474674701690674, 0.013592950068414211, 0.027647046372294426, 0.05575403571128845, 0.030779868364334106, 0.01613735221326351, 0.05745428055524826, 0.03968728333711624, 0.03406553715467453, 0.013110978528857231, 0.059776563197374344, 0.016043761745095253, 0.029699290171265602, 0.020566539838910103, 0.08347290754318237, 0.03070860728621483, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15561480820178986, 0.030323194339871407, 0.02106756716966629, 0.038600414991378784, 0.025089262053370476, 0.04719521105289459, 0.04255649819970131, 0.051662590354681015, 0.030368102714419365, 0.020133620128035545, 0.04255696386098862, 0.018511781468987465, 0.01867171935737133, 0.0414302833378315, 0.022022131830453873, 0.02106102742254734, 0.0416661836206913, 0.027249205857515335, 0.04048159345984459, 0.015536420047283173, 0.04219621419906616, 0.017007341608405113, 0.06220947951078415, 0.015703151002526283, 0.05443187430500984, 0.04092409461736679, 0.015729239210486412, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1521887332201004, 0.03232048079371452, 0.026277603581547737, 0.03215136379003525, 0.03708053380250931, 0.03843334689736366, 0.042185209691524506, 0.034409213811159134, 0.029296185821294785, 0.02475224994122982, 0.03461232781410217, 0.021795086562633514, 0.023419681936502457, 0.03386644273996353, 0.028580598533153534, 0.03440327197313309, 0.033156976103782654, 0.026710400357842445, 0.04213565215468407, 0.019021350890398026, 0.032785814255476, 0.02683998830616474, 0.03304455429315567, 0.02330264076590538, 0.03390035033226013, 0.03944924846291542, 0.021334253251552582, 0.042546432465314865, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10210391879081726, 0.048305340111255646, 0.027957914397120476, 0.028134863823652267, 0.04368893802165985, 0.03765960410237312, 0.04012182354927063, 0.03589581325650215, 0.024491794407367706, 0.019768042489886284, 0.03467409312725067, 0.015431461855769157, 0.032616548240184784, 0.03414339944720268, 0.027461368590593338, 0.020323220640420914, 0.03445841744542122, 0.015827562659978867, 0.030705846846103668, 0.020360933616757393, 0.03558044880628586, 0.034790296107530594, 0.03518148139119148, 0.028572604060173035, 0.04333385452628136, 0.03896389529109001, 0.017587700858712196, 0.04553375393152237, 0.04632510989904404, 0.0, 0.0, 0.0, 0.0], [0.1184174120426178, 0.026689782738685608, 0.02854764647781849, 0.03516455739736557, 0.03519093617796898, 0.030704878270626068, 0.042546890676021576, 0.023103507235646248, 0.025463124737143517, 0.029678549617528915, 0.028395220637321472, 0.019178256392478943, 0.02383357845246792, 0.027640020474791527, 0.03381752222776413, 0.0396401509642601, 0.027574418112635612, 0.021856164559721947, 0.03966765105724335, 0.03941834345459938, 0.02711562067270279, 0.030498063191771507, 0.034932829439640045, 0.0323205292224884, 0.027840541675686836, 0.03877578675746918, 0.021124795079231262, 0.03861142694950104, 0.032771822065114975, 0.019479982554912567, 0.0, 0.0, 0.0], [0.05433792620897293, 0.019247835502028465, 0.011686461977660656, 0.014118609949946404, 0.01390978042036295, 0.05736807361245155, 0.0345463827252388, 0.046650584787130356, 0.013026447966694832, 0.013271295465528965, 0.05952775105834007, 0.009063147008419037, 0.017364196479320526, 0.061355650424957275, 0.01263465266674757, 0.015145537443459034, 0.06425532698631287, 0.046270616352558136, 0.01955636776983738, 0.012543462216854095, 0.06913558393716812, 0.034823667258024216, 0.030005954205989838, 0.012261823751032352, 0.07985591888427734, 0.018996428698301315, 0.010722929611802101, 0.0865454226732254, 0.017174094915390015, 0.022144164890050888, 0.022453978657722473, 0.0, 0.0], [0.05849646031856537, 0.03437737002968788, 0.013603095896542072, 0.02417043223977089, 0.01900586113333702, 0.0654640942811966, 0.030622966587543488, 0.03573206812143326, 0.01885945349931717, 0.012241484597325325, 0.06282190978527069, 0.011835752986371517, 0.009552359580993652, 0.06431794911623001, 0.02158958464860916, 0.010912974365055561, 0.0666591003537178, 0.02399745211005211, 0.026596758514642715, 0.006153556052595377, 0.06984329968690872, 0.03723869100213051, 0.030046707019209862, 0.021686814725399017, 0.043640609830617905, 0.021209606900811195, 0.013490457087755203, 0.04632722586393356, 0.020434511825442314, 0.013362264260649681, 0.018568990752100945, 0.04714024439454079, 0.0], [0.1352868676185608, 0.02973337098956108, 0.02611132338643074, 0.03753575682640076, 0.038881585001945496, 0.0331667996942997, 0.04824390634894371, 0.03046831116080284, 0.026038208976387978, 0.024123696610331535, 0.027892732992768288, 0.02270994521677494, 0.024185286834836006, 0.02611454948782921, 0.02320966310799122, 0.025175290182232857, 0.024614792317152023, 0.02112315408885479, 0.02968505024909973, 0.01816747523844242, 0.023064401000738144, 0.02059807814657688, 0.034411393105983734, 0.021083395928144455, 0.02253883145749569, 0.0322815477848053, 0.013987251557409763, 0.02414492331445217, 0.02700231224298477, 0.012929446063935757, 0.03616539016366005, 0.02684023790061474, 0.03248502314090729]]]}
     )
@@ -748,7 +748,7 @@ <h3>Activation Patching on the Indirect Object Identification Task<a class="head
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area docutils container">
-<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "3829b426693a411babc613fc1d68b1c8"}</script></div>
+<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "8de918cf34204469a9a6abae1699ad36"}</script></div>
 </div>
 <p>We can now visualize the results, and see that this computation is extremely localised within the model. Initially, the second subject (Mary) token is all that matters (naturally, as it’s the only different token), and all relevant information remains here until heads in layer 7 and 8 move this to the final token where it’s used to predict the indirect object. (Note - the heads are in layer 7 and 8, not 8 and 9, because we patched in the residual stream at the <em>start</em> of each layer)</p>
 <div class="nbinput docutils container">
@@ -769,9 +769,9 @@ <h3>Activation Patching on the Indirect Object Identification Task<a class="head
 <head><meta charset="utf-8" /></head>
 <body>
     <div>            <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG"></script><script type="text/javascript">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: "STIX-Web"}});}</script>                <script type="text/javascript">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>
-        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="ac2799ee-2c40-4026-87e5-7bc5d779bc16" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("ac2799ee-2c40-4026-87e5-7bc5d779bc16")) {                    Plotly.newPlot(                        "ac2799ee-2c40-4026-87e5-7bc5d779bc16",                        [{"coloraxis":"coloraxis","name":"0","x":["\u003c|endoftext|\u003e_0","After_1"," John_2"," and_3"," Mary_4"," went_5"," to_6"," the_7"," store_8",",_9"," Mary_10"," gave_11"," a_12"," bottle_13"," of_14"," milk_15"," to_16"],"z":[[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9981481432914734,0.001600604155100882,0.0001503795647295192,-0.0003718698862940073,-2.257052983622998e-05,-0.000628303736448288,-0.000515179184731096],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9980564713478088,0.002283838577568531,0.00018205988453701138,-0.0005045737489126623,-0.0002681270125322044,-5.153151141712442e-05,-0.0012816254748031497],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9967373609542847,0.004081866703927517,0.0009732521721161902,4.378139055916108e-05,-0.00015962531324476004,-0.00033638247987255454,-0.0019441929180175066],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9905908107757568,0.019987020641565323,0.0018955166451632977,0.0010141782695427537,-6.75756236887537e-05,0.000911251176148653,-0.0019012272823601961],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9616515636444092,0.08534706383943558,0.0052040028385818005,0.003052052343264222,0.00019688029715325683,0.0011058200616389513,-0.002284654416143894],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.963099479675293,0.0843719094991684,0.00412211287766695,0.0007176341023296118,0.00010319898865418509,0.0010020771296694875,-0.004215250723063946],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9359182119369507,0.11111730337142944,0.007705116644501686,0.0003754050412680954,0.0003647996054496616,0.0013266304740682244,0.01874428242444992],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.7701544761657715,0.03741976246237755,0.0020680588204413652,-8.321183850057423e-05,0.00013447141100186855,0.0017246060306206346,0.44990694522857666],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0965060144662857,0.025925789028406143,0.0019711144268512726,0.00032904025283642113,0.00042367333662696183,0.00188477523624897,0.8994728922843933],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.023322155699133873,0.018537340685725212,0.001587143400683999,0.000526600400917232,0.0002531706413719803,0.0008727724780328572,0.9612765908241272],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.008559261448681355,0.006339463870972395,0.0005807152483612299,-0.000342636980349198,0.00010958944039884955,0.0006486988277174532,0.9495824575424194]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Position: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Position"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Normalized Logit Difference After Patching Residual Stream on the IOI Task"}},                        {"responsive": true}                    ).then(function(){
+        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="8ba8ca06-6030-4e88-87b2-a1ff0dfb3302" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("8ba8ca06-6030-4e88-87b2-a1ff0dfb3302")) {                    Plotly.newPlot(                        "8ba8ca06-6030-4e88-87b2-a1ff0dfb3302",                        [{"coloraxis":"coloraxis","name":"0","x":["\u003c|endoftext|\u003e_0","After_1"," John_2"," and_3"," Mary_4"," went_5"," to_6"," the_7"," store_8",",_9"," Mary_10"," gave_11"," a_12"," bottle_13"," of_14"," milk_15"," to_16"],"z":[[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9981481432914734,0.001600604155100882,0.0001503795647295192,-0.0003718698862940073,-2.257052983622998e-05,-0.000628303736448288,-0.000515179184731096],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9980564713478088,0.002283838577568531,0.00018205988453701138,-0.0005045737489126623,-0.0002681270125322044,-5.153151141712442e-05,-0.0012816254748031497],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9967373609542847,0.004081866703927517,0.0009732521721161902,4.378139055916108e-05,-0.00015962531324476004,-0.00033638247987255454,-0.0019441929180175066],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9905908107757568,0.019987020641565323,0.0018955166451632977,0.0010141782695427537,-6.75756236887537e-05,0.000911251176148653,-0.0019012272823601961],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9616515636444092,0.08534706383943558,0.0052040028385818005,0.003052052343264222,0.00019688029715325683,0.0011058200616389513,-0.002284654416143894],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.963099479675293,0.0843719094991684,0.00412211287766695,0.0007176341023296118,0.00010319898865418509,0.0010020771296694875,-0.004215250723063946],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9359182119369507,0.11111730337142944,0.007705116644501686,0.0003754050412680954,0.0003647996054496616,0.0013266304740682244,0.01874428242444992],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.7701544761657715,0.03741976246237755,0.0020680588204413652,-8.321183850057423e-05,0.00013447141100186855,0.0017246060306206346,0.44990694522857666],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0965060144662857,0.025925789028406143,0.0019711144268512726,0.00032904025283642113,0.00042367333662696183,0.00188477523624897,0.8994728922843933],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.023322155699133873,0.018537340685725212,0.001587143400683999,0.000526600400917232,0.0002531706413719803,0.0008727724780328572,0.9612765908241272],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.008559261448681355,0.006339463870972395,0.0005807152483612299,-0.000342636980349198,0.00010958944039884955,0.0006486988277174532,0.9495824575424194]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Position: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Position"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Normalized Logit Difference After Patching Residual Stream on the IOI Task"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('ac2799ee-2c40-4026-87e5-7bc5d779bc16');
+var gd = document.getElementById('8ba8ca06-6030-4e88-87b2-a1ff0dfb3302');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -833,9 +833,9 @@ <h2>Hooks: Accessing Activations<a class="headerlink" href="#Hooks:-Accessing-Ac
 <head><meta charset="utf-8" /></head>
 <body>
     <div>            <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG"></script><script type="text/javascript">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: "STIX-Web"}});}</script>                <script type="text/javascript">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>
-        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="b5559adb-b6df-4664-85da-aad13d04f74a" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("b5559adb-b6df-4664-85da-aad13d04f74a")) {                    Plotly.newPlot(                        "b5559adb-b6df-4664-85da-aad13d04f74a",                        [{"hovertemplate":"variable=0\u003cbr\u003eindex=%{x}\u003cbr\u003evalue=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"0","line":{"color":"#636efa","dash":"solid"},"marker":{"symbol":"circle"},"mode":"lines","name":"0","orientation":"v","showlegend":true,"x":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98],"xaxis":"x","y":[11.219001770019531,12.648847579956055,16.002866744995117,12.655248641967773,13.598546981811523,12.648005485534668,13.230877876281738,11.950650215148926,12.584773063659668,13.332685470581055,11.372512817382812,13.18463134765625,12.104430198669434,12.645095825195312,13.332456588745117,12.490678787231445,13.130325317382812,12.238739967346191,13.111671447753906,13.072357177734375,11.822067260742188,12.05682373046875,12.79888916015625,12.196507453918457,12.406768798828125,11.383073806762695,12.965502738952637,11.637373924255371,10.67663860321045,11.894469261169434,13.117510795593262,11.437427520751953,11.200849533081055,11.569291114807129,10.730561256408691,12.751376152038574,11.765420913696289,12.07258415222168,11.446783065795898,11.796368598937988,11.440250396728516,11.975069046020508,10.75130844116211,10.960607528686523,11.075111389160156,10.82270622253418,11.122757911682129,11.248815536499023,12.138458251953125,10.384830474853516,2.522568702697754,1.1026382446289062,0.7128584384918213,0.4818822741508484,0.31920477747917175,0.15357092022895813,0.13818064332008362,0.16000370681285858,0.23586773872375488,0.06869865953922272,0.21874837577342987,0.11845606565475464,0.17535646259784698,0.0625605583190918,0.027201101183891296,0.07039335370063782,0.03363990783691406,0.047253381460905075,0.06748445332050323,0.013322415761649609,0.027032632380723953,0.21386680006980896,0.041274793446063995,0.14630667865276337,0.01946776732802391,0.11438588798046112,0.04898519441485405,0.027135217562317848,0.05644571781158447,0.11561702191829681,0.1028728261590004,0.049143046140670776,0.018940147012472153,0.028789499774575233,0.04571650177240372,0.009683424606919289,0.049352310597896576,0.05664796754717827,0.18444031476974487,0.06125975772738457,0.25522834062576294,0.4084078371524811,0.0669250562787056,0.02292284369468689,0.15346066653728485,0.07391653954982758,0.06822908669710159,0.0359528474509716,0.07923130691051483],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"index"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"value"}},"legend":{"title":{"text":"variable"},"tracegroupgap":0},"title":{"text":"Loss by position on random repeated tokens"}},                        {"responsive": true}                    ).then(function(){
+        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="bf609849-a683-4501-98c0-5610402069af" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("bf609849-a683-4501-98c0-5610402069af")) {                    Plotly.newPlot(                        "bf609849-a683-4501-98c0-5610402069af",                        [{"hovertemplate":"variable=0\u003cbr\u003eindex=%{x}\u003cbr\u003evalue=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"0","line":{"color":"#636efa","dash":"solid"},"marker":{"symbol":"circle"},"mode":"lines","name":"0","orientation":"v","showlegend":true,"x":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98],"xaxis":"x","y":[11.916303634643555,14.13970947265625,13.97419548034668,13.20025634765625,12.98205280303955,12.23963737487793,13.160758972167969,12.960711479187012,12.692301750183105,11.697540283203125,13.213212966918945,12.825567245483398,11.889198303222656,10.807716369628906,12.493316650390625,12.748961448669434,11.875480651855469,11.771799087524414,12.129152297973633,12.251404762268066,10.839369773864746,12.363996505737305,11.972248077392578,12.573539733886719,12.151954650878906,11.586162567138672,12.556215286254883,11.899218559265137,11.944257736206055,11.11519718170166,11.516045570373535,11.446038246154785,12.16328239440918,11.093135833740234,11.868314743041992,10.437663078308105,12.832025527954102,11.429876327514648,10.88789176940918,10.855405807495117,11.266473770141602,11.21790885925293,11.649023056030273,11.6561861038208,12.299074172973633,11.253116607666016,11.278722763061523,11.215995788574219,10.433588027954102,10.897237777709961,3.044490098953247,1.069545030593872,1.016643762588501,0.4091116786003113,0.432170569896698,0.24384959042072296,0.7159929871559143,0.2306588888168335,0.14308522641658783,0.2239038646221161,0.06011886149644852,0.16535978019237518,0.2879369556903839,0.040467940270900726,0.2961328625679016,0.14453032612800598,0.10808869451284409,0.08187573403120041,0.09832978248596191,0.052005164325237274,0.04744909703731537,0.12793658673763275,0.08489024639129639,0.03772114962339401,0.04858517646789551,0.08634428679943085,0.10197576135396957,0.021571611985564232,0.09415853023529053,0.06979478150606155,0.16457553207874298,0.04652702435851097,0.009943359531462193,0.039533548057079315,0.10102002322673798,0.022349249571561813,0.18824195861816406,0.03961147740483284,0.024878712370991707,0.0205758698284626,0.05781855434179306,0.014255058951675892,0.0838586762547493,0.024006284773349762,0.9298907518386841,0.0951559990644455,0.03629877790808678,0.07244053483009338,0.044225141406059265],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"index"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"value"}},"legend":{"title":{"text":"variable"},"tracegroupgap":0},"title":{"text":"Loss by position on random repeated tokens"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('b5559adb-b6df-4664-85da-aad13d04f74a');
+var gd = document.getElementById('bf609849-a683-4501-98c0-5610402069af');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -915,9 +915,9 @@ <h2>Hooks: Accessing Activations<a class="headerlink" href="#Hooks:-Accessing-Ac
 <head><meta charset="utf-8" /></head>
 <body>
     <div>            <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG"></script><script type="text/javascript">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: "STIX-Web"}});}</script>                <script type="text/javascript">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>
-        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="abbc7990-fb5b-4a65-ba89-78a2073c73df" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("abbc7990-fb5b-4a65-ba89-78a2073c73df")) {                    Plotly.newPlot(                        "abbc7990-fb5b-4a65-ba89-78a2073c73df",                        [{"coloraxis":"coloraxis","name":"0","z":[[0.010287518613040447,7.07331855664961e-05,0.01022765040397644,9.369629765387799e-07,0.0001593400229467079,6.424106686608866e-05,0.009279504418373108,0.0007229120819829404,0.00916075985878706,0.009111087769269943,0.006602954585105181,0.0152360824868083],[0.0005463710404001176,0.00035243277670815587,0.0021286087576299906,0.014092736877501011,0.005032995715737343,0.010190492495894432,0.015489863231778145,0.012780369259417057,0.012919844128191471,0.01602115109562874,0.0064891609363257885,0.0004646908782888204],[0.004244121257215738,0.018242545425891876,0.0023879925720393658,0.0012670502765104175,0.013606147840619087,0.0018331799656152725,0.0033160552848130465,0.00845535658299923,0.0038152106571942568,0.0014914380153641105,0.0006890674121677876,0.01019816193729639],[0.01416884083300829,0.006844150368124247,0.002567051211372018,0.01341474149376154,0.021600881591439247,0.01306712906807661,0.001567754428833723,0.0013694057706743479,0.0052207414992153645,0.01306647714227438,0.009408000856637955,0.007939903065562248],[0.015263119712471962,0.013007179833948612,0.014736704528331757,0.008307430893182755,0.018053121864795685,0.01217469573020935,0.008762415498495102,0.002206446137279272,0.015867728739976883,0.013769634068012238,0.018624909222126007,9.13730691110004e-09],[0.47185665369033813,0.9086124897003174,0.01296070497483015,0.006677804049104452,0.009637922048568726,0.9290925860404968,0.01033448614180088,0.018294984474778175,0.02779943309724331,0.02766416408121586,0.017583714798092842,0.017298221588134766],[0.007537637371569872,0.017424091696739197,0.018900733441114426,0.01431836187839508,0.022236179560422897,0.011647127568721771,0.03073083981871605,0.011905021034181118,0.008411239832639694,0.9175557494163513,0.03838243708014488,0.013622810132801533],[0.01000579446554184,0.17460721731185913,0.8657189011573792,0.017993038520216942,0.01724315620958805,0.018139807507395744,0.04568207263946533,0.09614809602499008,0.01781415194272995,0.018681053072214127,0.9199892282485962,0.05816270038485527],[0.016772812232375145,0.4192027151584625,0.016147194430232048,0.05109066516160965,0.016613438725471497,0.012248117476701736,0.1683734506368637,0.011353541165590286,0.03254345804452896,0.031994905322790146,0.07066226005554199,0.022813519462943077],[0.2658080458641052,0.18101611733436584,0.1085599958896637,0.01112457923591137,0.09504234045743942,0.027309555560350418,0.47915005683898926,0.03030301257967949,0.05258383974432945,0.4882603585720062,0.015520771034061909,0.04620152711868286],[0.36585161089897156,0.5181708335876465,0.03953804820775986,0.1504061222076416,0.05925014615058899,0.014007902704179287,0.31749680638313293,0.4777964651584625,0.04831257089972496,0.014951827004551888,0.1641472578048706,0.2731083333492279],[0.017277132719755173,0.05256804823875427,0.037828266620635986,0.010270046070218086,0.03663536533713341,0.0975709855556488,0.05082761123776436,0.06714604049921036,0.007663114927709103,0.3211846351623535,0.3871070444583893,0.02293839119374752]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Induction Score by Head"}},                        {"responsive": true}                    ).then(function(){
+        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="d4826cd4-d822-446a-9486-7f010e673bbf" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("d4826cd4-d822-446a-9486-7f010e673bbf")) {                    Plotly.newPlot(                        "d4826cd4-d822-446a-9486-7f010e673bbf",                        [{"coloraxis":"coloraxis","name":"0","z":[[0.010405071079730988,8.505234291078523e-05,0.010846362449228764,1.542941731713654e-06,0.00015207800606731325,0.00016587963909842074,0.00962826982140541,0.0005811225273646414,0.00920275691896677,0.009271877817809582,0.006795060820877552,0.015113143250346184],[0.0007497637998312712,0.0003636373148765415,0.002115159062668681,0.014394569210708141,0.005006241612136364,0.010394004173576832,0.015367378480732441,0.01268749125301838,0.012691996991634369,0.016459191218018532,0.006484598852694035,0.0005559310666285455],[0.004768972750753164,0.01647953875362873,0.0027482376899570227,0.0013796724379062653,0.012924875132739544,0.0018674784805625677,0.003705371869727969,0.008166997693479061,0.004057642072439194,0.0012803683057427406,0.000662333972286433,0.010387937538325787],[0.015679387375712395,0.007625625003129244,0.002122837584465742,0.013503310270607471,0.02086609974503517,0.01211901567876339,0.001721711247228086,0.0009715322521515191,0.00451809773221612,0.01239688228815794,0.008685757406055927,0.006643661297857761],[0.016919558867812157,0.01504963357001543,0.014352007769048214,0.008405109867453575,0.01849125511944294,0.01381102204322815,0.009090271778404713,0.001800192752853036,0.013694416731595993,0.014873025007545948,0.018977493047714233,9.226908126436228e-09],[0.4363614022731781,0.9093427658081055,0.014157562516629696,0.007539841346442699,0.011399139650166035,0.9306832551956177,0.010265758261084557,0.017486363649368286,0.024969371035695076,0.027117766439914703,0.018408896401524544,0.01754666306078434],[0.009883408434689045,0.017079785466194153,0.0185578390955925,0.015524016693234444,0.020320380106568336,0.012695600278675556,0.030558481812477112,0.012052788399159908,0.01120459008961916,0.916867733001709,0.032972753047943115,0.01364062912762165],[0.008361412212252617,0.171825110912323,0.8461205363273621,0.018025416880846024,0.018446065485477448,0.01776888407766819,0.04303755983710289,0.08179712295532227,0.015604961663484573,0.018940657377243042,0.9122208952903748,0.05450733006000519],[0.014317817986011505,0.3731990456581116,0.014678257517516613,0.04617224633693695,0.017426906153559685,0.01191379688680172,0.1442830115556717,0.013355129398405552,0.029831815510988235,0.03103570081293583,0.06533283740282059,0.021598415449261665],[0.24887625873088837,0.1976928859949112,0.1104423925280571,0.012795306742191315,0.09626349806785583,0.02669103629887104,0.44758379459381104,0.028052648529410362,0.05248158425092697,0.4636542797088623,0.01666952669620514,0.041144587099552155],[0.3466327488422394,0.5092848539352417,0.03988264873623848,0.1478113979101181,0.06403890997171402,0.01558593474328518,0.29378482699394226,0.4561544358730316,0.052708882838487625,0.016264332458376884,0.1581176519393921,0.2725672125816345],[0.017380494624376297,0.05578174069523811,0.03640053793787956,0.009598813019692898,0.0362129844725132,0.10631337016820908,0.054979126900434494,0.06522571295499802,0.007772110868245363,0.3159135580062866,0.40592387318611145,0.021894147619605064]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Induction Score by Head"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('abbc7990-fb5b-4a65-ba89-78a2073c73df');
+var gd = document.getElementById('d4826cd4-d822-446a-9486-7f010e673bbf');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -985,11 +985,11 @@ <h2>Hooks: Accessing Activations<a class="headerlink" href="#Hooks:-Accessing-Ac
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area rendered_html docutils container">
-<div id="circuits-vis-34e21d87-69ed" style="margin: 15px 0;"/>
+<div id="circuits-vis-0896baec-5f5f" style="margin: 15px 0;"/>
     <script crossorigin type="module">
     import { render, AttentionPatterns } from "https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js";
     render(
-      "circuits-vis-34e21d87-69ed",
+      "circuits-vis-0896baec-5f5f",
       AttentionPatterns,
       {"tokens": ["use", " advice", " Social", "\u00f6", "\u00b7", " fought", " Le", " allegedly", " NO", "alth", "car", " prepared", "new", "rant", "roll", " hours", " published", "66", "ension", " 44", "use", " advice", " Social", "\u00f6", "\u00b7", " fought", " Le", " allegedly", " NO", "alth", "car", " prepared", "new", "rant", "roll", " hours", " published", "66", "ension", " 44"], "attention": [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9737270474433899, 0.02627299167215824, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9820428490638733, 0.017020218074321747, 0.0009368443279527128, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9895542860031128, 0.008665801025927067, 0.00041197543032467365, 0.0013679895782843232, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8543054461479187, 0.07801821082830429, 0.0008415375486947596, 0.0001359905581921339, 0.06669880449771881, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.937433660030365, 0.03300206735730171, 0.0015577399171888828, 2.535264457037556e-06, 0.0010925547685474157, 0.026911458000540733, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9769211411476135, 0.0038436795584857464, 2.234023304481525e-05, 3.5218730772612616e-05, 0.00518348254263401, 0.012176254764199257, 0.0018179015023633838, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9473504424095154, 0.013174979016184807, 0.0013492192374542356, 1.1802491826529149e-05, 0.0009449502103962004, 0.01131904125213623, 0.018021006137132645, 0.00782855786383152, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9847128391265869, 0.0010781444143503904, 0.0021734496112912893, 5.4822653510200325e-06, 0.0004914247547276318, 0.0013570826267823577, 0.00010185814608121291, 0.0002853855839930475, 0.009794436395168304, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9915198683738708, 0.004483341239392757, 0.00012727164721582085, 0.00016702155699022114, 0.0016301742289215326, 0.0011521608103066683, 0.00032312856637872756, 0.00012646272080019116, 0.00039313736488111317, 7.735046528978273e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8908806443214417, 0.024311941117048264, 1.7341310012852773e-05, 4.157719376962632e-05, 0.0008967609610408545, 0.07334917038679123, 0.0009482790483161807, 0.004280843771994114, 0.00516867870464921, 7.830337381165009e-06, 9.693232277641073e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8170808553695679, 0.1351710557937622, 0.011989916674792767, 1.1421708222769666e-05, 0.0003511959221214056, 0.009450665675103664, 0.01946333795785904, 0.0006557459710165858, 0.0005760999629274011, 2.9927030482213013e-05, 1.6589872757322155e-05, 0.00520310876891017, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9082695841789246, 0.006068154238164425, 0.013871855102479458, 0.0008237077272497118, 0.01190832071006298, 0.015542060136795044, 0.008354832418262959, 0.002078164601698518, 0.0013173240004107356, 0.0021398039534687996, 0.0039441632106900215, 0.0012376613449305296, 0.024444298818707466, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9479591250419617, 0.0021026728209108114, 0.011938490904867649, 0.00012338497617747635, 3.5374921480979538e-06, 0.00014498813834507018, 0.0005875456845387816, 2.5534465748933144e-05, 0.0013608969748020172, 0.0003395687963347882, 0.010076268576085567, 0.015790557488799095, 0.006346334703266621, 0.0032010283321142197, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9393549561500549, 0.006392320618033409, 0.0018427377799525857, 6.1162154452176765e-06, 0.00033587400685064495, 0.002051525516435504, 0.003801509505137801, 0.0012357983505353332, 0.00021948102221358567, 0.0003869360953103751, 5.012243491364643e-05, 0.008153212256729603, 0.026924701407551765, 0.002937993500381708, 0.0063067772425711155, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9339622259140015, 0.0017828107811510563, 0.005864636041224003, 0.0001995021157199517, 7.227134483400732e-05, 0.0014535371446982026, 0.0025924311485141516, 0.00048593789688311517, 0.002229832811281085, 0.00015120484749786556, 0.012292915023863316, 0.005057850386947393, 0.01236859429627657, 0.0039444840513169765, 0.006275098770856857, 0.011266663670539856, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8931334018707275, 0.0015468214405700564, 0.013001682236790657, 7.966378689161502e-06, 5.864375998498872e-05, 0.0008863414404913783, 0.00320208678022027, 3.214761454728432e-05, 0.00018022459698840976, 1.1455733329057693e-05, 7.600126264151186e-05, 0.00042027299059554935, 0.0016126197297126055, 0.028539275750517845, 0.01053555216640234, 0.025432037189602852, 0.021323570981621742, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9847024083137512, 0.000458244961919263, 0.00017220497829839587, 6.160992711556901e-07, 4.782819814863615e-06, 0.0005806140834465623, 0.0004461870121303946, 0.0004120134108234197, 0.0013038699980825186, 0.0003176067257300019, 6.994167779339477e-05, 0.0013941116631031036, 5.583090751315467e-05, 0.0009110373794101179, 0.00019558057829272002, 0.0003960288013331592, 0.0011691722320392728, 0.007409730460494757, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8922597765922546, 0.01028366107493639, 0.007569293025881052, 0.015225829556584358, 0.0006035296246409416, 0.0014377726474776864, 0.018397418782114983, 0.00018186635861638933, 0.002113566268235445, 3.803668369073421e-05, 0.009962561540305614, 0.003998206928372383, 0.0012666863622143865, 0.0021862699650228024, 0.0032670684158802032, 0.0015871950890868902, 0.01913381926715374, 0.008779395371675491, 0.0017080748220905662, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5637838244438171, 2.204169322794769e-05, 0.0003808305482380092, 1.393800062032824e-07, 4.306913226059805e-08, 0.00012883225281257182, 5.202722968533635e-05, 4.098194494872587e-06, 0.00043821518192999065, 1.0102487067342736e-05, 2.0490055248956196e-05, 0.00021747536084149033, 2.5250110411434434e-05, 2.129377389792353e-05, 0.0022071911953389645, 5.892770423088223e-05, 0.002418374642729759, 0.003277554176747799, 0.42604929208755493, 0.0008841048693284392, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14635005593299866, 0.45109108090400696, 0.027205264195799828, 0.0030082776211202145, 0.0007913765148259699, 0.0800931379199028, 0.005927647929638624, 0.0006846337346360087, 0.0021268464624881744, 0.0027747468557208776, 0.00023907337163109332, 0.002550546545535326, 0.0054934462532401085, 0.01583220437169075, 0.0003450022195465863, 0.0005726688541471958, 0.002175167202949524, 0.039043255150318146, 0.16982686519622803, 0.04120763763785362, 0.0026610149070620537, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15883015096187592, 0.09413880854845047, 0.6926899552345276, 4.776390778715722e-05, 8.085349691100419e-06, 0.00935556460171938, 0.0008445517742075026, 2.443764515192015e-06, 0.00013773982936982065, 1.1189692941115936e-06, 4.677353445003973e-06, 0.0003472122480161488, 0.002631494775414467, 0.0004504133830778301, 0.006463215220719576, 0.0005723336944356561, 0.0012668337440118194, 0.006402322091162205, 0.0018092951504513621, 0.006555411033332348, 0.00037914939457550645, 0.017061391845345497, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05072391405701637, 0.0043561081402003765, 0.00013167233555577695, 0.9396469593048096, 0.0005500844563357532, 0.002771171974018216, 1.4556275345967151e-05, 5.0173721319879405e-06, 1.5498681023018435e-05, 7.020712899930004e-08, 8.694933967490215e-06, 3.654160536825657e-05, 3.607992539400584e-06, 2.5941648345906287e-05, 7.590860604977934e-06, 7.10092763256398e-07, 4.6298235247377306e-05, 7.143522088881582e-05, 0.00012089123629266396, 0.0005609994404949248, 1.3380947166297119e-05, 0.0007342157769016922, 0.0001547116116853431, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.044663090258836746, 0.0028326627798378468, 7.648707105545327e-05, 0.00015513667312916368, 0.7502228021621704, 0.19195663928985596, 9.640512871555984e-05, 0.00016210127796512097, 0.00012769461318384856, 1.1226569768041372e-05, 8.733285540074576e-06, 0.0002813320606946945, 5.2078037697356194e-05, 0.008386553265154362, 4.340289706306066e-06, 6.482724711531773e-05, 3.802955325227231e-05, 7.603636913700029e-05, 0.00012636324390769005, 9.227949340129271e-05, 4.0301659964825376e-07, 0.00011281677143415436, 2.522385329939425e-06, 0.0004493836604524404, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.013606224209070206, 0.006971816532313824, 3.1886105716694146e-05, 1.8455255030858098e-06, 0.002301084576174617, 0.9711790084838867, 0.0003632458101492375, 8.45977701828815e-05, 0.0001061156508512795, 4.5056845010549296e-07, 2.987579534874385e-07, 0.0008592646918259561, 8.844918920658529e-05, 0.00034812072408385575, 9.285179203288862e-07, 3.160578853567131e-05, 1.2802144738088828e-05, 5.8033951063407585e-05, 0.00010517514601815492, 6.438296259148046e-05, 1.8867377775677596e-06, 0.0009238758939318359, 4.68101461592596e-06, 8.99309179658303e-06, 0.0028452184051275253, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.01961127668619156, 0.00438615120947361, 6.198724440764636e-05, 4.88576041846045e-08, 4.252470171195455e-05, 0.0036122461315244436, 0.9598402380943298, 0.005622219294309616, 0.0023445591796189547, 5.173985186956997e-07, 1.9621675164671615e-06, 0.00165482924785465, 0.0005915391957387328, 0.001169584458693862, 5.784227596450364e-06, 8.119036647258326e-05, 4.500302384258248e-05, 0.00018497802375350147, 5.086949386168271e-05, 0.00011132832878502086, 6.848460998298833e-06, 3.8539848901564255e-05, 4.853054633713327e-06, 1.276350758416811e-07, 3.953097802877892e-06, 0.0005269531393423676, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.008729088120162487, 0.0001781070022843778, 2.8806286422877747e-07, 4.138263136610476e-07, 6.864999886602163e-05, 0.0009331137989647686, 0.0001488832349423319, 0.984452486038208, 0.004786369390785694, 0.0001132894612965174, 7.255491141222592e-07, 7.424689101753756e-05, 1.4996417121437844e-05, 0.00019790598889812827, 2.995051318066544e-07, 4.872014415013837e-06, 5.296082690620096e-06, 4.536818778433371e-06, 0.00011268883827142417, 3.4170998333138414e-06, 8.272462764580268e-06, 1.23798881759285e-05, 3.665505587946427e-08, 9.0251188566981e-07, 1.6053050785558298e-05, 0.00011404338874854147, 1.8653627193998545e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04082155600190163, 0.003618426388129592, 4.821046968572773e-05, 1.9615674773376668e-07, 5.3802777983946726e-05, 0.0021802401170134544, 0.0033272523432970047, 0.001878200564533472, 0.9307873249053955, 0.00481536565348506, 1.443025394110009e-05, 0.0028287956956773996, 0.00010464123624842614, 0.0029309086967259645, 0.0009514776756986976, 6.522179319290444e-05, 0.00029541170806623995, 0.00012317505024839193, 0.0015512284589931369, 0.0005327722756192088, 0.0002208336372859776, 0.00044327162322588265, 1.9391492969589308e-05, 5.825700100103859e-07, 1.6632819097139873e-05, 0.0004773235123138875, 0.0011258182348683476, 0.0007676867535337806, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.13800986111164093, 0.000338585494318977, 0.00015972793335095048, 3.422598027214008e-08, 1.2696453268290497e-05, 0.00024196873710025102, 3.82194593839813e-05, 3.751534677576274e-05, 0.004743305500596762, 0.8406771421432495, 0.00018875481327995658, 0.00015232685836963356, 1.250763853022363e-05, 0.0002236300497315824, 0.0001328253565588966, 6.470848165918142e-05, 0.00013950491847936064, 9.763600246515125e-05, 0.0004387965309433639, 4.486108446144499e-05, 6.358909013215452e-05, 0.00034256128128618, 5.2226907428121194e-05, 5.293519507176825e-07, 4.435913979250472e-06, 6.543510244227946e-05, 2.9145055577828316e-06, 3.8726907405362e-06, 0.013709785416722298, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0077987853437662125, 0.00022834629635326564, 4.638521033939469e-07, 6.367737910295546e-07, 1.5816805898793973e-05, 4.74391745228786e-05, 5.2391105782589875e-06, 7.306586212507682e-06, 1.252215042768512e-05, 3.330249569444277e-07, 0.9909055233001709, 0.00045220632455311716, 5.365398919821018e-06, 7.535894837928936e-05, 8.799969691608567e-06, 7.895076123531908e-06, 0.00023918210354167968, 2.256911329823197e-06, 7.641898992005736e-05, 5.514193617273122e-05, 4.378300673124613e-06, 2.181093987019267e-05, 4.435156242266203e-08, 1.2584397381942836e-06, 2.069822357952944e-06, 1.2951757526025176e-05, 3.952872305035271e-07, 1.1202362202311633e-06, 7.981805538292974e-06, 2.9978705242683645e-06, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04228433594107628, 0.01130534615367651, 7.640720696144854e-07, 2.2586132217838895e-06, 9.786387818166986e-05, 0.03826921060681343, 0.00023640895960852504, 0.001455514575354755, 0.0029497891664505005, 1.2863508800364798e-06, 2.8271693736314774e-05, 0.8946183323860168, 0.0005235012504272163, 0.0011799612548202276, 0.0009106284123845398, 0.0003604615631047636, 0.00020461656094994396, 2.910211514972616e-05, 0.0013425340875983238, 0.0003398252301849425, 0.00033254953450523317, 8.430417801719159e-05, 7.937666879342942e-08, 5.80987489229301e-06, 4.91085984322126e-06, 0.002387275919318199, 2.696716364880558e-05, 0.00020931514154653996, 0.0007688859477639198, 5.801381576020503e-06, 3.4147800761274993e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0061029596254229546, 0.016550516709685326, 4.6485445636790246e-05, 8.233629245069096e-08, 7.398499292321503e-06, 0.0005482283304445446, 0.0005931411869823933, 1.5738387446617708e-05, 3.503288098727353e-05, 3.3504278462714865e-07, 1.8657722478110372e-07, 0.00046066963113844395, 0.8416223526000977, 0.12311305105686188, 0.006359235383570194, 0.0026995246298611164, 0.00016676251834724098, 0.001213077805005014, 0.00013198796659708023, 8.468224405078217e-05, 4.615935722540598e-06, 6.994968862272799e-05, 4.916098987450823e-06, 3.4761444567266153e-07, 3.7119539797458856e-07, 6.107360968599096e-05, 6.728529115207493e-05, 2.2758390514354687e-06, 2.0601644791895524e-05, 7.198171942945919e-07, 2.78813168108627e-08, 1.6529560525668785e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04041679948568344, 0.001346431439742446, 0.0002881708787754178, 4.343534783401992e-06, 0.0006837420514784753, 0.0021492524538189173, 0.0012713877949863672, 0.00021145616483408958, 0.0001491332659497857, 2.3681526727159508e-05, 9.255093027604744e-05, 7.460857159458101e-05, 0.0030381479300558567, 0.9153464436531067, 0.002063496969640255, 0.002670764224603772, 0.0006971358670853078, 0.022915009409189224, 0.0016386171337217093, 0.0002943731960840523, 8.301850357383955e-06, 0.0001194091746583581, 3.610825660871342e-05, 1.2593762221513316e-05, 0.00038307331851683557, 0.0011952053755521774, 0.0001844509970396757, 7.125815318431705e-05, 0.00012850550410803407, 6.062198372092098e-05, 4.517644993029535e-05, 6.895749606883328e-07, 0.002379077021032572, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.03775661438703537, 0.0016132340533658862, 0.0003177856560796499, 8.17116472262569e-07, 4.6315386725837016e-07, 7.222060230560601e-05, 0.00012490959488786757, 4.99756515637273e-06, 0.00030667142709717155, 5.749376668973127e-06, 0.00024661491625010967, 0.003777766600251198, 0.0013358162250369787, 0.002113183494657278, 0.8997159600257874, 0.04417850822210312, 0.00033094940590672195, 0.0013849412789568305, 0.00039355995249934494, 0.002321068197488785, 0.000464222306618467, 0.00019271834753453732, 0.0005703889764845371, 5.927758593315957e-06, 2.4687206945372964e-08, 3.1083311569091165e-06, 7.165991064539412e-06, 2.9240266030683415e-07, 7.984187686815858e-05, 1.2546481229946949e-05, 0.00012651427823584527, 0.00023838109336793423, 0.0008907323936000466, 0.0014062426052987576, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08771780878305435, 0.00782146118581295, 5.6624849094077945e-05, 1.4731161179781793e-08, 1.0490798558748793e-05, 0.0011058745440095663, 0.00042911627679131925, 5.52300589333754e-05, 3.919872688129544e-05, 2.7502203465701314e-06, 2.5461979475949192e-06, 0.002755212364718318, 0.005722680129110813, 0.0011191823286935687, 0.0015876379329711199, 0.8757358193397522, 0.001080039655789733, 0.0035321444738656282, 0.0015019761631265283, 0.0006949526141397655, 4.334090044721961e-05, 0.00023435342882294208, 6.934279099368723e-06, 2.757802519681718e-08, 9.912479299600818e-07, 0.00019156596681568772, 2.9515393180190586e-05, 5.468214112624992e-06, 2.480577677488327e-06, 1.4659925682281028e-06, 7.256068101924029e-07, 8.567866461817175e-05, 0.004263878334313631, 0.0007048743427731097, 0.0034579597413539886, 0.0, 0.0, 0.0, 0.0, 0.0], [0.018741386011242867, 0.0001317748101428151, 0.00011519038525875658, 8.336405130648927e-07, 3.873432774526009e-07, 4.0343722503166646e-05, 8.927313319873065e-05, 6.575927727681119e-06, 6.646558176726103e-05, 2.0145262169535272e-07, 0.00012498456635512412, 0.00022737613471690565, 0.00035967829171568155, 4.087335037183948e-05, 8.65399488247931e-05, 0.00017062197730410844, 0.9720673561096191, 0.004511047620326281, 0.0017741499468684196, 0.00033421028638258576, 0.0006673677125945687, 8.272402737929951e-06, 8.590817014919594e-05, 4.445200545433181e-07, 1.5395107411109166e-08, 3.916216883226298e-06, 1.8561571778263897e-05, 1.7440333976992406e-06, 2.306471833435353e-05, 6.561277814398636e-07, 5.536517346627079e-05, 1.0027780263044406e-05, 0.00011005577107425779, 2.0988292817492038e-05, 6.0086065786890686e-05, 4.435596929397434e-05, 0.0, 0.0, 0.0, 0.0], [0.02161449007689953, 6.699986261082813e-05, 0.00023383108782581985, 1.9461008093912824e-07, 1.05680453543755e-06, 2.569968273746781e-05, 0.00021046775509603322, 1.7352506347378949e-06, 6.270146968745394e-06, 8.374834692403965e-08, 2.354792513870052e-06, 1.806468026188668e-05, 0.00023376732133328915, 0.0018953320104628801, 0.00020409566059242934, 0.0004746932827401906, 0.0029953878838568926, 0.962553858757019, 0.0069142491556704044, 0.0001976252970052883, 0.00011761276255128905, 1.7810549252317287e-05, 8.193789835786447e-05, 8.319970135062249e-08, 6.341087299688297e-08, 1.5034723901408142e-06, 1.5951758541632444e-05, 1.285200426082156e-07, 7.935356052257703e-07, 1.2686608386047737e-07, 5.14019859565451e-07, 3.882940191601847e-08, 5.885375867364928e-05, 0.0011026912834495306, 9.71672561718151e-05, 8.231383981183171e-05, 0.0007721207803115249, 0.0, 0.0, 0.0], [0.036683376878499985, 5.292138666845858e-06, 1.6796333284219145e-06, 3.225949229967995e-10, 1.7679223418554102e-08, 9.711849997984245e-06, 6.97010455041891e-06, 3.922056293959031e-06, 1.5272446034941822e-05, 6.423179002013057e-07, 3.00263849339899e-07, 2.1527906937990338e-05, 8.620364724265528e-07, 2.37110707530519e-05, 1.133400132857787e-06, 4.3700861169782e-06, 2.1794843632960692e-05, 0.0007244828739203513, 0.956550121307373, 7.606161670992151e-05, 0.005361158866435289, 3.409740747883916e-05, 1.4788669204790494e-06, 1.967607188291254e-09, 6.515270101203896e-09, 7.016132030912559e-07, 1.9082037283624231e-07, 3.6050772678208887e-07, 3.973362709075445e-06, 1.5304426597140264e-06, 1.0371004321996224e-07, 7.635152314833249e-07, 1.8985033989338262e-07, 6.164461410662625e-06, 2.613814103824552e-06, 1.352905542262306e-06, 1.0274186024616938e-05, 0.0004238302353769541, 0.0, 0.0], [0.03937007114291191, 0.00035857962211593986, 7.419281610054895e-05, 4.950996662955731e-05, 4.6530603867722675e-06, 4.697596523328684e-05, 0.00022992271988186985, 1.2504364121923572e-06, 3.72327740478795e-05, 8.911823101698246e-08, 7.931985601317137e-05, 0.000135091133415699, 1.8374767023487948e-05, 0.00010297342669218779, 8.208450162783265e-05, 2.753637636487838e-05, 0.0005808338755741715, 0.0011750266421586275, 0.0009059829753823578, 0.9545682072639465, 7.417569577228278e-05, 0.0005086683668196201, 0.00019448318926151842, 8.633160177851096e-05, 8.95263099209842e-07, 1.4289948921941686e-05, 3.445189940975979e-05, 2.1258644267163618e-07, 1.6950663848547265e-05, 4.816170076082926e-07, 3.18538659485057e-05, 1.3589556147053372e-05, 2.5087487301789224e-05, 5.9819085436174646e-05, 0.0001169012684840709, 1.1341999197611585e-05, 0.0002713195572141558, 0.0002638357982505113, 0.00042732004658319056, 0.0], [0.025401176884770393, 7.441661864504567e-07, 1.0830090104718693e-05, 2.2497226304096785e-09, 9.221913788159952e-10, 8.208573490264826e-06, 2.5448041469644522e-06, 1.0004280426301193e-07, 3.160869891871698e-05, 2.4278301680169534e-07, 5.289186333357065e-07, 1.4262926924857311e-05, 1.5345807469202555e-06, 1.87778505278402e-06, 0.0002028680028161034, 2.1412402020359877e-06, 0.00010114445467479527, 0.0009309852030128241, 0.1016610637307167, 5.551521462621167e-05, 0.8627558350563049, 2.250279521831544e-06, 3.066586941713467e-05, 5.196949981467469e-09, 3.1991624627192294e-11, 8.736405021636529e-08, 1.2241659419487405e-07, 1.2374152813521277e-09, 1.2507486644608434e-05, 4.635763275473437e-07, 1.6297862259762042e-07, 2.719466181133612e-07, 1.1941818911509472e-07, 1.4462575848028791e-07, 5.527890243683942e-05, 3.5639450857161137e-07, 4.933351738145575e-05, 0.0001694788079475984, 0.008456701412796974, 3.871364606311545e-05]]]}
     )
@@ -1015,37 +1015,37 @@ <h2>Available Models<a class="headerlink" href="#Available-Models" title="Permal
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area docutils container">
-<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "43da9025ada5462c9ceacafb0040d9e5"}</script></div>
+<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "70c3c51f506a4de4974cf045dc40e642"}</script></div>
 </div>
 <div class="nboutput docutils container">
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area docutils container">
-<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "b9efbbee427945918852a8f8351f3658"}</script></div>
+<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "c2024b930e0f46828f9f1935807c38fd"}</script></div>
 </div>
 <div class="nboutput docutils container">
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area docutils container">
-<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "cad219cb55d34a84a8bd505e58157885"}</script></div>
+<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "0688c83589ce4772b743149fd2bfabce"}</script></div>
 </div>
 <div class="nboutput docutils container">
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area docutils container">
-<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "e60d70eb78324541b8f9e1d0766491c8"}</script></div>
+<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "27f33facb9e34151b8dce0f220cad270"}</script></div>
 </div>
 <div class="nboutput docutils container">
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area docutils container">
-<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "40e09b2a289644d4b6031bd159605d65"}</script></div>
+<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "f9a0ca4b543f47e0987f20fc87d7135a"}</script></div>
 </div>
 <div class="nboutput docutils container">
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area docutils container">
-<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "8bccf23ea8614f7790636e7dd69b28b0"}</script></div>
+<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "e4a807366560448bab5fc25c6deea451"}</script></div>
 </div>
 <div class="nboutput nblast docutils container">
 <div class="prompt empty docutils container">
@@ -1097,9 +1097,9 @@ <h2>Available Models<a class="headerlink" href="#Available-Models" title="Permal
 <head><meta charset="utf-8" /></head>
 <body>
     <div>            <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG"></script><script type="text/javascript">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: "STIX-Web"}});}</script>                <script type="text/javascript">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>
-        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="80cfda6e-58b4-4243-84ab-714b7a85733a" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("80cfda6e-58b4-4243-84ab-714b7a85733a")) {                    Plotly.newPlot(                        "80cfda6e-58b4-4243-84ab-714b7a85733a",                        [{"coloraxis":"coloraxis","name":"0","z":[[0.010475716553628445,0.00012220541248098016,0.010917986743152142,7.656216439499985e-06,0.0006968218949623406,6.9313814492488746e-06,0.0084159504622221,0.0012861847644671798,0.008170148357748985,0.009381728246808052,0.008698969148099422,0.01542614120990038],[0.002591925673186779,0.01701531372964382,0.00246937433257699,0.00014253608242142946,0.014017364010214806,0.0016980216605588794,0.003427345072850585,0.015447111800312996,0.002018758561462164,0.0001262403529835865,0.003834882751107216,0.014691304415464401],[0.009293507784605026,0.006629703566431999,0.013332263566553593,0.0016038173343986273,0.019033372402191162,0.005262553226202726,0.009104946628212929,0.0015506644267588854,0.01918281428515911,0.012545435689389706,0.02117963135242462,4.0281530888641726e-10],[0.005776463076472282,0.22343513369560242,0.858056902885437,0.01594151370227337,0.016118988394737244,0.012186223641037941,0.019889742136001587,0.2014574259519577,0.01657862216234207,0.017072241753339767,0.935918390750885,0.4967102110385895],[0.2777242362499237,0.2320401519536972,0.09087100625038147,0.010066336952149868,0.08514240384101868,0.024723602458834648,0.645492672920227,0.024798421189188957,0.06863012909889221,0.655184268951416,0.015323049388825893,0.059535764157772064],[0.01975579373538494,0.07690141350030899,0.05422816053032875,0.01160797756165266,0.03362690284848213,0.1715722680091858,0.0725213959813118,0.09838436543941498,0.006914828438311815,0.4509258270263672,0.17343828082084656,0.030602477490901947]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Induction Score by Head in Distil GPT-2"}},                        {"responsive": true}                    ).then(function(){
+        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="7a2572ee-0c07-4383-838b-a1506ba287c1" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("7a2572ee-0c07-4383-838b-a1506ba287c1")) {                    Plotly.newPlot(                        "7a2572ee-0c07-4383-838b-a1506ba287c1",                        [{"coloraxis":"coloraxis","name":"0","z":[[0.010156218893826008,0.0001236683747265488,0.011424270458519459,7.034143436612794e-06,0.000634899246506393,1.026747213472845e-05,0.008508010767400265,0.0013871823903173208,0.008700739592313766,0.009379616938531399,0.009299803525209427,0.015903957188129425],[0.0028657428920269012,0.015641430392861366,0.003444988513365388,0.0002817462373059243,0.013710059225559235,0.0021904385648667812,0.0045652128756046295,0.01538788340985775,0.00303249410353601,7.614650530740619e-05,0.003943697549402714,0.014629906043410301],[0.011179465800523758,0.007786225993186235,0.0122522609308362,0.001650955411605537,0.018960842862725258,0.004585776012390852,0.010203308425843716,0.0018027113983407617,0.017543736845254898,0.013146407902240753,0.021970190107822418,5.426245824197151e-12],[0.0065704951994121075,0.21819067001342773,0.8631006479263306,0.015738708898425102,0.017175206914544106,0.013265381567180157,0.01839497685432434,0.18099769949913025,0.014968963339924812,0.016049183905124664,0.9444844722747803,0.4698825478553772],[0.2739366590976715,0.2408347725868225,0.08438918739557266,0.011110114865005016,0.08356543630361557,0.024493325501680374,0.6308431625366211,0.024440230801701546,0.0658520832657814,0.6499699950218201,0.017077801749110222,0.05483752116560936],[0.019521579146385193,0.07837866246700287,0.05102621018886566,0.01296086423099041,0.032427575439214706,0.18894724547863007,0.07455243170261383,0.0973595529794693,0.007094149943441153,0.4619784951210022,0.1653374880552292,0.02855134755373001]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0},"title":{"text":"Induction Score by Head in Distil GPT-2"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('80cfda6e-58b4-4243-84ab-714b7a85733a');
+var gd = document.getElementById('7a2572ee-0c07-4383-838b-a1506ba287c1');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -1552,20 +1552,12 @@ <h2>Dealing with tokens<a class="headerlink" href="#Dealing-with-tokens" title="
 </pre></div>
 </div>
 </div>
-<div class="nboutput docutils container">
-<div class="prompt empty docutils container">
-</div>
-<div class="output_area docutils container">
-<div class="highlight"><pre>
-Probability tensor shape [batch, position, d_vocab] == torch.Size([1, 8, 50257])
-</pre></div></div>
-</div>
 <div class="nboutput nblast docutils container">
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area docutils container">
 <div class="highlight"><pre>
-
+Probability tensor shape [batch, position, d_vocab] == torch.Size([1, 8, 50257])
 | The| probability: 11.98%
 </pre></div></div>
 </div>
@@ -1939,9 +1931,9 @@ <h3>Medium Example: Eigenvalue Copying Scores<a class="headerlink" href="#Medium
 <head><meta charset="utf-8" /></head>
 <body>
     <div>            <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG"></script><script type="text/javascript">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: "STIX-Web"}});}</script>                <script type="text/javascript">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>
-        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="f0e72e2d-ef0f-47a5-a93a-ab60fc607f1b" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("f0e72e2d-ef0f-47a5-a93a-ab60fc607f1b")) {                    Plotly.newPlot(                        "f0e72e2d-ef0f-47a5-a93a-ab60fc607f1b",                        [{"coloraxis":"coloraxis","name":"0","z":[[0.7775008678436279,0.35272663831710815,0.25961872935295105,0.6670258045196533,0.838425874710083,0.5584433078765869,0.844474732875824,0.41379114985466003,0.244889497756958,0.02815753035247326,0.3584096431732178,0.16288283467292786],[-0.4541913866996765,-0.6529324650764465,-0.5484569668769836,-0.7990368604660034,-0.7736426591873169,-0.8522582650184631,0.977432370185852,0.6626251935958862,-0.7303223609924316,-0.7007019519805908,-0.6946623921394348,-0.9996723532676697],[-0.7837162613868713,0.8967758417129517,0.4750956892967224,-0.6671973466873169,0.7881463170051575,-0.8547748923301697,-0.9054183959960938,-0.5749384760856628,-0.321751207113266,-0.0285941194742918,-0.9247617125511169,-0.9699268937110901],[0.5864036083221436,-0.76143479347229,0.5971695184707642,0.7854391932487488,-0.8788884878158569,0.3908745348453522,0.0447387658059597,0.1102800965309143,-0.8169987201690674,0.2212953418493271,-0.9939578771591187,0.5774401426315308],[0.5254791378974915,0.30490121245384216,-0.10729170590639114,0.9433152675628662,-0.9314427971839905,0.5273631811141968,-0.4264712929725647,-0.9984429478645325,0.5296757817268372,0.8604294657707214,-0.8895052075386047,0.9556970000267029],[0.6629188656806946,0.4295697808265686,0.9736858010292053,0.655548095703125,0.12201863527297974,0.7442769408226013,0.5037954449653625,0.95253586769104,-0.6507166624069214,-0.9316278100013733,0.9791510701179504,-0.9972586035728455],[0.9613031148910522,0.7501779794692993,-0.3806658983230591,0.6429785490036011,0.9557769298553467,-0.9428837895393372,-0.9948079586029053,0.7852989435195923,0.9657301306724548,0.7073014974594116,0.36872273683547974,0.8128010034561157],[0.9659482836723328,0.9730120301246643,0.31900620460510254,-0.30290529131889343,0.9790952801704407,0.9357922673225403,-0.5550314784049988,-0.0054661668837070465,0.986777663230896,0.8249568343162537,0.5664296746253967,0.1000528484582901],[-0.9464486837387085,-0.25471991300582886,0.6522327065467834,0.14152583479881287,0.9884141683578491,0.9860584735870361,0.6949271559715271,0.9901811480522156,0.9791203141212463,-0.23595543205738068,-0.982071042060852,0.6506688594818115],[0.9895945191383362,-0.291781485080719,0.9714024662971497,0.9951602220535278,0.18783727288246155,-0.9460937976837158,0.4780191481113434,-0.2489192634820938,0.9437099099159241,0.11866225302219391,0.9941242933273315,-0.38088199496269226],[0.9564487338066101,0.5542722344398499,0.42118069529533386,0.6628788113594055,0.8659593462944031,0.9937117695808411,0.9069075584411621,0.3981107473373413,-0.4134218096733093,0.9971914887428284,0.3459664583206177,0.9938657283782959],[0.5891268253326416,0.9313738942146301,0.9268401861190796,0.9993563890457153,0.6227542161941528,0.8463947772979736,0.6584343910217285,0.8423123955726624,0.2978499233722687,0.8728678822517395,0.9963143467903137,0.9867526292800903]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0,"cmin":-1.0,"cmax":1.0},"title":{"text":"OV Copying Score for each head in GPT-2 Small"}},                        {"responsive": true}                    ).then(function(){
+        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="83f1fdc6-5e71-4ccb-8cbf-7af2e7ebf238" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("83f1fdc6-5e71-4ccb-8cbf-7af2e7ebf238")) {                    Plotly.newPlot(                        "83f1fdc6-5e71-4ccb-8cbf-7af2e7ebf238",                        [{"coloraxis":"coloraxis","name":"0","z":[[0.7775008678436279,0.35272663831710815,0.25961872935295105,0.6670258045196533,0.838425874710083,0.5584433078765869,0.844474732875824,0.41379114985466003,0.244889497756958,0.02815753035247326,0.3584096431732178,0.16288283467292786],[-0.4541913866996765,-0.6529324650764465,-0.5484569668769836,-0.7990368604660034,-0.7736426591873169,-0.8522582650184631,0.977432370185852,0.6626251935958862,-0.7303223609924316,-0.7007019519805908,-0.6946623921394348,-0.9996723532676697],[-0.7837162613868713,0.8967758417129517,0.4750956892967224,-0.6671973466873169,0.7881463170051575,-0.8547748923301697,-0.9054183959960938,-0.5749384760856628,-0.321751207113266,-0.0285941194742918,-0.9247617125511169,-0.9699268937110901],[0.5864036083221436,-0.76143479347229,0.5971695184707642,0.7854391932487488,-0.8788884878158569,0.3908745348453522,0.0447387658059597,0.1102800965309143,-0.8169987201690674,0.2212953418493271,-0.9939578771591187,0.5774401426315308],[0.5254791378974915,0.30490121245384216,-0.10729170590639114,0.9433152675628662,-0.9314427971839905,0.5273631811141968,-0.4264712929725647,-0.9984429478645325,0.5296757817268372,0.8604294657707214,-0.8895052075386047,0.9556970000267029],[0.6629188656806946,0.4295697808265686,0.9736858010292053,0.655548095703125,0.12201863527297974,0.7442769408226013,0.5037954449653625,0.95253586769104,-0.6507166624069214,-0.9316278100013733,0.9791510701179504,-0.9972586035728455],[0.9613031148910522,0.7501779794692993,-0.3806658983230591,0.6429785490036011,0.9557769298553467,-0.9428837895393372,-0.9948079586029053,0.7852989435195923,0.9657301306724548,0.7073014974594116,0.36872273683547974,0.8128010034561157],[0.9659482836723328,0.9730120301246643,0.31900620460510254,-0.30290529131889343,0.9790952801704407,0.9357922673225403,-0.5550314784049988,-0.0054661668837070465,0.986777663230896,0.8249568343162537,0.5664296746253967,0.1000528484582901],[-0.9464486837387085,-0.25471991300582886,0.6522327065467834,0.14152583479881287,0.9884141683578491,0.9860584735870361,0.6949271559715271,0.9901811480522156,0.9791203141212463,-0.23595543205738068,-0.982071042060852,0.6506688594818115],[0.9895945191383362,-0.291781485080719,0.9714024662971497,0.9951602220535278,0.18783727288246155,-0.9460937976837158,0.4780191481113434,-0.2489192634820938,0.9437099099159241,0.11866225302219391,0.9941242933273315,-0.38088199496269226],[0.9564487338066101,0.5542722344398499,0.42118069529533386,0.6628788113594055,0.8659593462944031,0.9937117695808411,0.9069075584411621,0.3981107473373413,-0.4134218096733093,0.9971914887428284,0.3459664583206177,0.9938657283782959],[0.5891268253326416,0.9313738942146301,0.9268401861190796,0.9993563890457153,0.6227542161941528,0.8463947772979736,0.6584343910217285,0.8423123955726624,0.2978499233722687,0.8728678822517395,0.9963143467903137,0.9867526292800903]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0,"cmin":-1.0,"cmax":1.0},"title":{"text":"OV Copying Score for each head in GPT-2 Small"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('f0e72e2d-ef0f-47a5-a93a-ab60fc607f1b');
+var gd = document.getElementById('83f1fdc6-5e71-4ccb-8cbf-7af2e7ebf238');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -1984,9 +1976,9 @@ <h3>Medium Example: Eigenvalue Copying Scores<a class="headerlink" href="#Medium
 <head><meta charset="utf-8" /></head>
 <body>
     <div>            <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG"></script><script type="text/javascript">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: "STIX-Web"}});}</script>                <script type="text/javascript">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>
-        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="eb9ce2dc-9731-481d-a530-6a91763cefe7" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("eb9ce2dc-9731-481d-a530-6a91763cefe7")) {                    Plotly.newPlot(                        "eb9ce2dc-9731-481d-a530-6a91763cefe7",                        [{"hovertemplate":"Real=%{x}\u003cbr\u003eImaginary=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"","marker":{"color":"#636efa","symbol":"circle"},"mode":"markers","name":"","orientation":"v","showlegend":false,"x":[-2.139726161956787,1.4152659177780151,3.4444546699523926,4.0276689529418945,8.882657051086426,4.866776943206787,4.866776943206787,4.843708038330078,4.843708038330078,8.477534294128418,8.21679973602295,8.21679973602295,5.078621864318848,7.855459213256836,7.855459213256836,5.365771770477295,5.365771770477295,5.563427448272705,5.563427448272705,5.421727657318115,7.769131660461426,7.769131660461426,7.042290210723877,7.042290210723877,5.675149917602539,5.675149917602539,7.6785993576049805,7.6785993576049805,6.573331356048584,6.573331356048584,7.67292594909668,7.172202110290527,7.172202110290527,7.423615455627441,7.423615455627441,7.4708147048950195,6.08908748626709,6.08908748626709,6.306832313537598,6.306832313537598,6.511747360229492,6.511747360229492,5.955250263214111,5.955250263214111,5.858802795410156,5.858802795410156,7.147878170013428,7.147878170013428,7.185699939727783,7.185699939727783,6.6706037521362305,6.6706037521362305,6.7359819412231445,6.7359819412231445,6.149754524230957,6.149754524230957,6.288780212402344,6.288780212402344,6.344789028167725,6.625576972961426,6.625576972961426,6.899187088012695,6.899187088012695,6.856410026550293],"xaxis":"x","y":[0.0,0.0,0.0,0.0,0.0,0.41851967573165894,-0.41851967573165894,0.09079498052597046,-0.09079498052597046,0.0,0.40868431329727173,-0.40868431329727173,0.0,0.7007191777229309,-0.7007191777229309,0.46421778202056885,-0.46421778202056885,0.5558270215988159,-0.5558270215988159,0.0,0.47056713700294495,-0.47056713700294495,1.029872179031372,-1.029872179031372,0.48252731561660767,-0.48252731561660767,0.33564886450767517,-0.33564886450767517,0.9988701343536377,-0.9988701343536377,0.0,0.7531797885894775,-0.7531797885894775,0.425758957862854,-0.425758957862854,0.0,0.6436269283294678,-0.6436269283294678,0.7701666951179504,-0.7701666951179504,0.7558029294013977,-0.7558029294013977,0.2591177523136139,-0.2591177523136139,0.013043581508100033,-0.013043581508100033,0.40166139602661133,-0.40166139602661133,0.28192487359046936,-0.28192487359046936,0.6146230697631836,-0.6146230697631836,0.5391324758529663,-0.5391324758529663,0.2823374569416046,-0.2823374569416046,0.35283783078193665,-0.35283783078193665,0.0,0.24868342280387878,-0.24868342280387878,0.15545885264873505,-0.15545885264873505,0.0],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"Real"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"Imaginary"}},"legend":{"tracegroupgap":0},"title":{"text":"Eigenvalues of Head L11H11 of GPT-2 Small"}},                        {"responsive": true}                    ).then(function(){
+        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="25ecddbf-c1bd-497d-b838-6c0e162ebacb" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("25ecddbf-c1bd-497d-b838-6c0e162ebacb")) {                    Plotly.newPlot(                        "25ecddbf-c1bd-497d-b838-6c0e162ebacb",                        [{"hovertemplate":"Real=%{x}\u003cbr\u003eImaginary=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"","marker":{"color":"#636efa","symbol":"circle"},"mode":"markers","name":"","orientation":"v","showlegend":false,"x":[-2.139726161956787,1.4152659177780151,3.4444546699523926,4.0276689529418945,8.882657051086426,4.866776943206787,4.866776943206787,4.843708038330078,4.843708038330078,8.477534294128418,8.21679973602295,8.21679973602295,5.078621864318848,7.855459213256836,7.855459213256836,5.365771770477295,5.365771770477295,5.563427448272705,5.563427448272705,5.421727657318115,7.769131660461426,7.769131660461426,7.042290210723877,7.042290210723877,5.675149917602539,5.675149917602539,7.6785993576049805,7.6785993576049805,6.573331356048584,6.573331356048584,7.67292594909668,7.172202110290527,7.172202110290527,7.423615455627441,7.423615455627441,7.4708147048950195,6.08908748626709,6.08908748626709,6.306832313537598,6.306832313537598,6.511747360229492,6.511747360229492,5.955250263214111,5.955250263214111,5.858802795410156,5.858802795410156,7.147878170013428,7.147878170013428,7.185699939727783,7.185699939727783,6.6706037521362305,6.6706037521362305,6.7359819412231445,6.7359819412231445,6.149754524230957,6.149754524230957,6.288780212402344,6.288780212402344,6.344789028167725,6.625576972961426,6.625576972961426,6.899187088012695,6.899187088012695,6.856410026550293],"xaxis":"x","y":[0.0,0.0,0.0,0.0,0.0,0.41851967573165894,-0.41851967573165894,0.09079498052597046,-0.09079498052597046,0.0,0.40868431329727173,-0.40868431329727173,0.0,0.7007191777229309,-0.7007191777229309,0.46421778202056885,-0.46421778202056885,0.5558270215988159,-0.5558270215988159,0.0,0.47056713700294495,-0.47056713700294495,1.029872179031372,-1.029872179031372,0.48252731561660767,-0.48252731561660767,0.33564886450767517,-0.33564886450767517,0.9988701343536377,-0.9988701343536377,0.0,0.7531797885894775,-0.7531797885894775,0.425758957862854,-0.425758957862854,0.0,0.6436269283294678,-0.6436269283294678,0.7701666951179504,-0.7701666951179504,0.7558029294013977,-0.7558029294013977,0.2591177523136139,-0.2591177523136139,0.013043581508100033,-0.013043581508100033,0.40166139602661133,-0.40166139602661133,0.28192487359046936,-0.28192487359046936,0.6146230697631836,-0.6146230697631836,0.5391324758529663,-0.5391324758529663,0.2823374569416046,-0.2823374569416046,0.35283783078193665,-0.35283783078193665,0.0,0.24868342280387878,-0.24868342280387878,0.15545885264873505,-0.15545885264873505,0.0],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"Real"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"Imaginary"}},"legend":{"tracegroupgap":0},"title":{"text":"Eigenvalues of Head L11H11 of GPT-2 Small"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('eb9ce2dc-9731-481d-a530-6a91763cefe7');
+var gd = document.getElementById('25ecddbf-c1bd-497d-b838-6c0e162ebacb');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -2066,9 +2058,9 @@ <h3>Medium Example: Eigenvalue Copying Scores<a class="headerlink" href="#Medium
 <head><meta charset="utf-8" /></head>
 <body>
     <div>            <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG"></script><script type="text/javascript">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: "STIX-Web"}});}</script>                <script type="text/javascript">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>
-        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="e3552224-0615-4466-b566-225b4371075b" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("e3552224-0615-4466-b566-225b4371075b")) {                    Plotly.newPlot(                        "e3552224-0615-4466-b566-225b4371075b",                        [{"coloraxis":"coloraxis","name":"0","z":[[0.8356367349624634,0.5853534936904907,0.5105841159820557,0.7843377590179443,0.8644161224365234,0.7026589512825012,0.8969924449920654,0.5868823528289795,0.42486509680747986,-0.16337518393993378,0.46268585324287415,0.276053786277771],[-0.052920110523700714,-0.31773144006729126,-0.4810579717159271,-0.7838066220283508,-0.6360211968421936,-0.77586829662323,0.9681803584098816,0.8119116425514221,-0.7510464787483215,-0.6878445744514465,-0.6429887413978577,-0.9985856413841248],[-0.6598325967788696,0.9152501821517944,0.5461500883102417,-0.48743969202041626,0.7720563411712646,-0.7541061639785767,-0.8472450971603394,-0.6948987245559692,-0.1557510495185852,0.2444225549697876,-0.9106622338294983,-0.9439151287078857],[0.648689866065979,-0.5592910647392273,0.5935593247413635,0.7843040823936462,-0.8150346279144287,0.613004744052887,0.16785882413387299,0.35195910930633545,-0.6837262511253357,0.2223764955997467,-0.9929219484329224,0.6535818576812744],[0.5740953087806702,0.3640134036540985,0.09609051048755646,0.9359624981880188,-0.9228776097297668,0.6191076636314392,-0.33572638034820557,-0.998464822769165,0.6448632478713989,0.8468661308288574,-0.7557656764984131,0.9527971744537354],[0.7326544523239136,0.5324169397354126,0.9732670187950134,0.7239246964454651,0.2553895115852356,0.8158416152000427,0.665579080581665,0.9287100434303284,-0.5660436153411865,-0.8908745050430298,0.9834234118461609,-0.9981179237365723],[0.9698692560195923,0.7439671754837036,-0.35639333724975586,0.6022987365722656,0.9708116054534912,-0.9278275966644287,-0.9962316155433655,0.834520697593689,0.9714328050613403,0.8158544898033142,0.5902575850486755,0.8199343681335449],[0.9820227026939392,0.9859329462051392,0.5152460932731628,-0.5610517263412476,0.9663666486740112,0.9495157599449158,-0.5204814076423645,0.3104752004146576,0.9859083890914917,0.7797460556030273,0.6738530397415161,0.39197394251823425],[-0.906204104423523,0.11750960350036621,0.8077874779701233,0.416930615901947,0.9829014539718628,0.9902300834655762,0.7847104668617249,0.9945629835128784,0.9868024587631226,-0.26804426312446594,-0.9908867478370667,0.745792806148529],[0.9906190633773804,-0.18231146037578583,0.9757838249206543,0.9986750483512878,0.2544330954551697,-0.954406201839447,0.586924135684967,-0.23537978529930115,0.9550502896308899,0.2551196813583374,0.9929869771003723,0.0905260294675827],[0.9707273244857788,0.6956090927124023,0.6280022263526917,0.7902868390083313,0.9343841671943665,0.9895793795585632,0.9436282515525818,-0.10834990441799164,-0.3431110680103302,0.9986709952354431,0.508673906326294,0.9949509501457214],[0.8283132910728455,0.9432437419891357,0.9491764903068542,0.9995353817939758,0.5712320804595947,0.8055236339569092,0.6781865954399109,0.8272573351860046,0.8314797282218933,0.8778655529022217,0.9944959282875061,0.997386634349823]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0,"cmin":-1.0,"cmax":1.0},"title":{"text":"OV Copying Score for each head in GPT-2 Small"}},                        {"responsive": true}                    ).then(function(){
+        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="039423ef-6937-4f4c-a9b7-5f7f1ec8d99c" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("039423ef-6937-4f4c-a9b7-5f7f1ec8d99c")) {                    Plotly.newPlot(                        "039423ef-6937-4f4c-a9b7-5f7f1ec8d99c",                        [{"coloraxis":"coloraxis","name":"0","z":[[0.8356367349624634,0.5853534936904907,0.5105841159820557,0.7843377590179443,0.8644161224365234,0.7026589512825012,0.8969924449920654,0.5868823528289795,0.42486509680747986,-0.16337518393993378,0.46268585324287415,0.276053786277771],[-0.052920110523700714,-0.31773144006729126,-0.4810579717159271,-0.7838066220283508,-0.6360211968421936,-0.77586829662323,0.9681803584098816,0.8119116425514221,-0.7510464787483215,-0.6878445744514465,-0.6429887413978577,-0.9985856413841248],[-0.6598325967788696,0.9152501821517944,0.5461500883102417,-0.48743969202041626,0.7720563411712646,-0.7541061639785767,-0.8472450971603394,-0.6948987245559692,-0.1557510495185852,0.2444225549697876,-0.9106622338294983,-0.9439151287078857],[0.648689866065979,-0.5592910647392273,0.5935593247413635,0.7843040823936462,-0.8150346279144287,0.613004744052887,0.16785882413387299,0.35195910930633545,-0.6837262511253357,0.2223764955997467,-0.9929219484329224,0.6535818576812744],[0.5740953087806702,0.3640134036540985,0.09609051048755646,0.9359624981880188,-0.9228776097297668,0.6191076636314392,-0.33572638034820557,-0.998464822769165,0.6448632478713989,0.8468661308288574,-0.7557656764984131,0.9527971744537354],[0.7326544523239136,0.5324169397354126,0.9732670187950134,0.7239246964454651,0.2553895115852356,0.8158416152000427,0.665579080581665,0.9287100434303284,-0.5660436153411865,-0.8908745050430298,0.9834234118461609,-0.9981179237365723],[0.9698692560195923,0.7439671754837036,-0.35639333724975586,0.6022987365722656,0.9708116054534912,-0.9278275966644287,-0.9962316155433655,0.834520697593689,0.9714328050613403,0.8158544898033142,0.5902575850486755,0.8199343681335449],[0.9820227026939392,0.9859329462051392,0.5152460932731628,-0.5610517263412476,0.9663666486740112,0.9495157599449158,-0.5204814076423645,0.3104752004146576,0.9859083890914917,0.7797460556030273,0.6738530397415161,0.39197394251823425],[-0.906204104423523,0.11750960350036621,0.8077874779701233,0.416930615901947,0.9829014539718628,0.9902300834655762,0.7847104668617249,0.9945629835128784,0.9868024587631226,-0.26804426312446594,-0.9908867478370667,0.745792806148529],[0.9906190633773804,-0.18231146037578583,0.9757838249206543,0.9986750483512878,0.2544330954551697,-0.954406201839447,0.586924135684967,-0.23537978529930115,0.9550502896308899,0.2551196813583374,0.9929869771003723,0.0905260294675827],[0.9707273244857788,0.6956090927124023,0.6280022263526917,0.7902868390083313,0.9343841671943665,0.9895793795585632,0.9436282515525818,-0.10834990441799164,-0.3431110680103302,0.9986709952354431,0.508673906326294,0.9949509501457214],[0.8283132910728455,0.9432437419891357,0.9491764903068542,0.9995353817939758,0.5712320804595947,0.8055236339569092,0.6781865954399109,0.8272573351860046,0.8314797282218933,0.8778655529022217,0.9944959282875061,0.997386634349823]],"type":"heatmap","xaxis":"x","yaxis":"y","hovertemplate":"Head: %{x}\u003cbr\u003eLayer: %{y}\u003cbr\u003ecolor: %{z}\u003cextra\u003e\u003c\u002fextra\u003e"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"scaleanchor":"y","constrain":"domain","title":{"text":"Head"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"autorange":"reversed","constrain":"domain","title":{"text":"Layer"}},"coloraxis":{"colorscale":[[0.0,"rgb(103,0,31)"],[0.1,"rgb(178,24,43)"],[0.2,"rgb(214,96,77)"],[0.3,"rgb(244,165,130)"],[0.4,"rgb(253,219,199)"],[0.5,"rgb(247,247,247)"],[0.6,"rgb(209,229,240)"],[0.7,"rgb(146,197,222)"],[0.8,"rgb(67,147,195)"],[0.9,"rgb(33,102,172)"],[1.0,"rgb(5,48,97)"]],"cmid":0.0,"cmin":-1.0,"cmax":1.0},"title":{"text":"OV Copying Score for each head in GPT-2 Small"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('e3552224-0615-4466-b566-225b4371075b');
+var gd = document.getElementById('039423ef-6937-4f4c-a9b7-5f7f1ec8d99c');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -2111,9 +2103,9 @@ <h3>Medium Example: Eigenvalue Copying Scores<a class="headerlink" href="#Medium
 <head><meta charset="utf-8" /></head>
 <body>
     <div>            <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG"></script><script type="text/javascript">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: "STIX-Web"}});}</script>                <script type="text/javascript">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>
-        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="2be56e5c-d2b2-481c-94a5-bddccdf1e7cd" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("2be56e5c-d2b2-481c-94a5-bddccdf1e7cd")) {                    Plotly.newPlot(                        "2be56e5c-d2b2-481c-94a5-bddccdf1e7cd",                        [{"hovertemplate":"\u003cb\u003e%{hovertext}\u003c\u002fb\u003e\u003cbr\u003e\u003cbr\u003eFull OV Copying Score=%{x}\u003cbr\u003eOV Copying Score=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","hovertext":["L0H0","L0H1","L0H2","L0H3","L0H4","L0H5","L0H6","L0H7","L0H8","L0H9","L0H10","L0H11","L1H0","L1H1","L1H2","L1H3","L1H4","L1H5","L1H6","L1H7","L1H8","L1H9","L1H10","L1H11","L2H0","L2H1","L2H2","L2H3","L2H4","L2H5","L2H6","L2H7","L2H8","L2H9","L2H10","L2H11","L3H0","L3H1","L3H2","L3H3","L3H4","L3H5","L3H6","L3H7","L3H8","L3H9","L3H10","L3H11","L4H0","L4H1","L4H2","L4H3","L4H4","L4H5","L4H6","L4H7","L4H8","L4H9","L4H10","L4H11","L5H0","L5H1","L5H2","L5H3","L5H4","L5H5","L5H6","L5H7","L5H8","L5H9","L5H10","L5H11","L6H0","L6H1","L6H2","L6H3","L6H4","L6H5","L6H6","L6H7","L6H8","L6H9","L6H10","L6H11","L7H0","L7H1","L7H2","L7H3","L7H4","L7H5","L7H6","L7H7","L7H8","L7H9","L7H10","L7H11","L8H0","L8H1","L8H2","L8H3","L8H4","L8H5","L8H6","L8H7","L8H8","L8H9","L8H10","L8H11","L9H0","L9H1","L9H2","L9H3","L9H4","L9H5","L9H6","L9H7","L9H8","L9H9","L9H10","L9H11","L10H0","L10H1","L10H2","L10H3","L10H4","L10H5","L10H6","L10H7","L10H8","L10H9","L10H10","L10H11","L11H0","L11H1","L11H2","L11H3","L11H4","L11H5","L11H6","L11H7","L11H8","L11H9","L11H10","L11H11"],"legendgroup":"","marker":{"color":"#636efa","symbol":"circle"},"mode":"markers","name":"","orientation":"v","showlegend":false,"x":[0.8356367349624634,0.5853534936904907,0.5105841159820557,0.7843377590179443,0.8644161224365234,0.7026589512825012,0.8969924449920654,0.5868823528289795,0.42486509680747986,-0.16337518393993378,0.46268585324287415,0.276053786277771,-0.052920110523700714,-0.31773144006729126,-0.4810579717159271,-0.7838066220283508,-0.6360211968421936,-0.77586829662323,0.9681803584098816,0.8119116425514221,-0.7510464787483215,-0.6878445744514465,-0.6429887413978577,-0.9985856413841248,-0.6598325967788696,0.9152501821517944,0.5461500883102417,-0.48743969202041626,0.7720563411712646,-0.7541061639785767,-0.8472450971603394,-0.6948987245559692,-0.1557510495185852,0.2444225549697876,-0.9106622338294983,-0.9439151287078857,0.648689866065979,-0.5592910647392273,0.5935593247413635,0.7843040823936462,-0.8150346279144287,0.613004744052887,0.16785882413387299,0.35195910930633545,-0.6837262511253357,0.2223764955997467,-0.9929219484329224,0.6535818576812744,0.5740953087806702,0.3640134036540985,0.09609051048755646,0.9359624981880188,-0.9228776097297668,0.6191076636314392,-0.33572638034820557,-0.998464822769165,0.6448632478713989,0.8468661308288574,-0.7557656764984131,0.9527971744537354,0.7326544523239136,0.5324169397354126,0.9732670187950134,0.7239246964454651,0.2553895115852356,0.8158416152000427,0.665579080581665,0.9287100434303284,-0.5660436153411865,-0.8908745050430298,0.9834234118461609,-0.9981179237365723,0.9698692560195923,0.7439671754837036,-0.35639333724975586,0.6022987365722656,0.9708116054534912,-0.9278275966644287,-0.9962316155433655,0.834520697593689,0.9714328050613403,0.8158544898033142,0.5902575850486755,0.8199343681335449,0.9820227026939392,0.9859329462051392,0.5152460932731628,-0.5610517263412476,0.9663666486740112,0.9495157599449158,-0.5204814076423645,0.3104752004146576,0.9859083890914917,0.7797460556030273,0.6738530397415161,0.39197394251823425,-0.906204104423523,0.11750960350036621,0.8077874779701233,0.416930615901947,0.9829014539718628,0.9902300834655762,0.7847104668617249,0.9945629835128784,0.9868024587631226,-0.26804426312446594,-0.9908867478370667,0.745792806148529,0.9906190633773804,-0.18231146037578583,0.9757838249206543,0.9986750483512878,0.2544330954551697,-0.954406201839447,0.586924135684967,-0.23537978529930115,0.9550502896308899,0.2551196813583374,0.9929869771003723,0.0905260294675827,0.9707273244857788,0.6956090927124023,0.6280022263526917,0.7902868390083313,0.9343841671943665,0.9895793795585632,0.9436282515525818,-0.10834990441799164,-0.3431110680103302,0.9986709952354431,0.508673906326294,0.9949509501457214,0.8283132910728455,0.9432437419891357,0.9491764903068542,0.9995353817939758,0.5712320804595947,0.8055236339569092,0.6781865954399109,0.8272573351860046,0.8314797282218933,0.8778655529022217,0.9944959282875061,0.997386634349823],"xaxis":"x","y":[0.7775008678436279,0.35272663831710815,0.25961872935295105,0.6670258045196533,0.838425874710083,0.5584433078765869,0.844474732875824,0.41379114985466003,0.244889497756958,0.02815753035247326,0.3584096431732178,0.16288283467292786,-0.4541913866996765,-0.6529324650764465,-0.5484569668769836,-0.7990368604660034,-0.7736426591873169,-0.8522582650184631,0.977432370185852,0.6626251935958862,-0.7303223609924316,-0.7007019519805908,-0.6946623921394348,-0.9996723532676697,-0.7837162613868713,0.8967758417129517,0.4750956892967224,-0.6671973466873169,0.7881463170051575,-0.8547748923301697,-0.9054183959960938,-0.5749384760856628,-0.321751207113266,-0.0285941194742918,-0.9247617125511169,-0.9699268937110901,0.5864036083221436,-0.76143479347229,0.5971695184707642,0.7854391932487488,-0.8788884878158569,0.3908745348453522,0.0447387658059597,0.1102800965309143,-0.8169987201690674,0.2212953418493271,-0.9939578771591187,0.5774401426315308,0.5254791378974915,0.30490121245384216,-0.10729170590639114,0.9433152675628662,-0.9314427971839905,0.5273631811141968,-0.4264712929725647,-0.9984429478645325,0.5296757817268372,0.8604294657707214,-0.8895052075386047,0.9556970000267029,0.6629188656806946,0.4295697808265686,0.9736858010292053,0.655548095703125,0.12201863527297974,0.7442769408226013,0.5037954449653625,0.95253586769104,-0.6507166624069214,-0.9316278100013733,0.9791510701179504,-0.9972586035728455,0.9613031148910522,0.7501779794692993,-0.3806658983230591,0.6429785490036011,0.9557769298553467,-0.9428837895393372,-0.9948079586029053,0.7852989435195923,0.9657301306724548,0.7073014974594116,0.36872273683547974,0.8128010034561157,0.9659482836723328,0.9730120301246643,0.31900620460510254,-0.30290529131889343,0.9790952801704407,0.9357922673225403,-0.5550314784049988,-0.0054661668837070465,0.986777663230896,0.8249568343162537,0.5664296746253967,0.1000528484582901,-0.9464486837387085,-0.25471991300582886,0.6522327065467834,0.14152583479881287,0.9884141683578491,0.9860584735870361,0.6949271559715271,0.9901811480522156,0.9791203141212463,-0.23595543205738068,-0.982071042060852,0.6506688594818115,0.9895945191383362,-0.291781485080719,0.9714024662971497,0.9951602220535278,0.18783727288246155,-0.9460937976837158,0.4780191481113434,-0.2489192634820938,0.9437099099159241,0.11866225302219391,0.9941242933273315,-0.38088199496269226,0.9564487338066101,0.5542722344398499,0.42118069529533386,0.6628788113594055,0.8659593462944031,0.9937117695808411,0.9069075584411621,0.3981107473373413,-0.4134218096733093,0.9971914887428284,0.3459664583206177,0.9938657283782959,0.5891268253326416,0.9313738942146301,0.9268401861190796,0.9993563890457153,0.6227542161941528,0.8463947772979736,0.6584343910217285,0.8423123955726624,0.2978499233722687,0.8728678822517395,0.9963143467903137,0.9867526292800903],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"Full OV Copying Score"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"OV Copying Score"}},"legend":{"tracegroupgap":0},"title":{"text":"OV Copying Score for each head in GPT-2 Small"}},                        {"responsive": true}                    ).then(function(){
+        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="2386d5b5-f55c-48a9-850d-e7c0f3dfdb0b" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("2386d5b5-f55c-48a9-850d-e7c0f3dfdb0b")) {                    Plotly.newPlot(                        "2386d5b5-f55c-48a9-850d-e7c0f3dfdb0b",                        [{"hovertemplate":"\u003cb\u003e%{hovertext}\u003c\u002fb\u003e\u003cbr\u003e\u003cbr\u003eFull OV Copying Score=%{x}\u003cbr\u003eOV Copying Score=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","hovertext":["L0H0","L0H1","L0H2","L0H3","L0H4","L0H5","L0H6","L0H7","L0H8","L0H9","L0H10","L0H11","L1H0","L1H1","L1H2","L1H3","L1H4","L1H5","L1H6","L1H7","L1H8","L1H9","L1H10","L1H11","L2H0","L2H1","L2H2","L2H3","L2H4","L2H5","L2H6","L2H7","L2H8","L2H9","L2H10","L2H11","L3H0","L3H1","L3H2","L3H3","L3H4","L3H5","L3H6","L3H7","L3H8","L3H9","L3H10","L3H11","L4H0","L4H1","L4H2","L4H3","L4H4","L4H5","L4H6","L4H7","L4H8","L4H9","L4H10","L4H11","L5H0","L5H1","L5H2","L5H3","L5H4","L5H5","L5H6","L5H7","L5H8","L5H9","L5H10","L5H11","L6H0","L6H1","L6H2","L6H3","L6H4","L6H5","L6H6","L6H7","L6H8","L6H9","L6H10","L6H11","L7H0","L7H1","L7H2","L7H3","L7H4","L7H5","L7H6","L7H7","L7H8","L7H9","L7H10","L7H11","L8H0","L8H1","L8H2","L8H3","L8H4","L8H5","L8H6","L8H7","L8H8","L8H9","L8H10","L8H11","L9H0","L9H1","L9H2","L9H3","L9H4","L9H5","L9H6","L9H7","L9H8","L9H9","L9H10","L9H11","L10H0","L10H1","L10H2","L10H3","L10H4","L10H5","L10H6","L10H7","L10H8","L10H9","L10H10","L10H11","L11H0","L11H1","L11H2","L11H3","L11H4","L11H5","L11H6","L11H7","L11H8","L11H9","L11H10","L11H11"],"legendgroup":"","marker":{"color":"#636efa","symbol":"circle"},"mode":"markers","name":"","orientation":"v","showlegend":false,"x":[0.8356367349624634,0.5853534936904907,0.5105841159820557,0.7843377590179443,0.8644161224365234,0.7026589512825012,0.8969924449920654,0.5868823528289795,0.42486509680747986,-0.16337518393993378,0.46268585324287415,0.276053786277771,-0.052920110523700714,-0.31773144006729126,-0.4810579717159271,-0.7838066220283508,-0.6360211968421936,-0.77586829662323,0.9681803584098816,0.8119116425514221,-0.7510464787483215,-0.6878445744514465,-0.6429887413978577,-0.9985856413841248,-0.6598325967788696,0.9152501821517944,0.5461500883102417,-0.48743969202041626,0.7720563411712646,-0.7541061639785767,-0.8472450971603394,-0.6948987245559692,-0.1557510495185852,0.2444225549697876,-0.9106622338294983,-0.9439151287078857,0.648689866065979,-0.5592910647392273,0.5935593247413635,0.7843040823936462,-0.8150346279144287,0.613004744052887,0.16785882413387299,0.35195910930633545,-0.6837262511253357,0.2223764955997467,-0.9929219484329224,0.6535818576812744,0.5740953087806702,0.3640134036540985,0.09609051048755646,0.9359624981880188,-0.9228776097297668,0.6191076636314392,-0.33572638034820557,-0.998464822769165,0.6448632478713989,0.8468661308288574,-0.7557656764984131,0.9527971744537354,0.7326544523239136,0.5324169397354126,0.9732670187950134,0.7239246964454651,0.2553895115852356,0.8158416152000427,0.665579080581665,0.9287100434303284,-0.5660436153411865,-0.8908745050430298,0.9834234118461609,-0.9981179237365723,0.9698692560195923,0.7439671754837036,-0.35639333724975586,0.6022987365722656,0.9708116054534912,-0.9278275966644287,-0.9962316155433655,0.834520697593689,0.9714328050613403,0.8158544898033142,0.5902575850486755,0.8199343681335449,0.9820227026939392,0.9859329462051392,0.5152460932731628,-0.5610517263412476,0.9663666486740112,0.9495157599449158,-0.5204814076423645,0.3104752004146576,0.9859083890914917,0.7797460556030273,0.6738530397415161,0.39197394251823425,-0.906204104423523,0.11750960350036621,0.8077874779701233,0.416930615901947,0.9829014539718628,0.9902300834655762,0.7847104668617249,0.9945629835128784,0.9868024587631226,-0.26804426312446594,-0.9908867478370667,0.745792806148529,0.9906190633773804,-0.18231146037578583,0.9757838249206543,0.9986750483512878,0.2544330954551697,-0.954406201839447,0.586924135684967,-0.23537978529930115,0.9550502896308899,0.2551196813583374,0.9929869771003723,0.0905260294675827,0.9707273244857788,0.6956090927124023,0.6280022263526917,0.7902868390083313,0.9343841671943665,0.9895793795585632,0.9436282515525818,-0.10834990441799164,-0.3431110680103302,0.9986709952354431,0.508673906326294,0.9949509501457214,0.8283132910728455,0.9432437419891357,0.9491764903068542,0.9995353817939758,0.5712320804595947,0.8055236339569092,0.6781865954399109,0.8272573351860046,0.8314797282218933,0.8778655529022217,0.9944959282875061,0.997386634349823],"xaxis":"x","y":[0.7775008678436279,0.35272663831710815,0.25961872935295105,0.6670258045196533,0.838425874710083,0.5584433078765869,0.844474732875824,0.41379114985466003,0.244889497756958,0.02815753035247326,0.3584096431732178,0.16288283467292786,-0.4541913866996765,-0.6529324650764465,-0.5484569668769836,-0.7990368604660034,-0.7736426591873169,-0.8522582650184631,0.977432370185852,0.6626251935958862,-0.7303223609924316,-0.7007019519805908,-0.6946623921394348,-0.9996723532676697,-0.7837162613868713,0.8967758417129517,0.4750956892967224,-0.6671973466873169,0.7881463170051575,-0.8547748923301697,-0.9054183959960938,-0.5749384760856628,-0.321751207113266,-0.0285941194742918,-0.9247617125511169,-0.9699268937110901,0.5864036083221436,-0.76143479347229,0.5971695184707642,0.7854391932487488,-0.8788884878158569,0.3908745348453522,0.0447387658059597,0.1102800965309143,-0.8169987201690674,0.2212953418493271,-0.9939578771591187,0.5774401426315308,0.5254791378974915,0.30490121245384216,-0.10729170590639114,0.9433152675628662,-0.9314427971839905,0.5273631811141968,-0.4264712929725647,-0.9984429478645325,0.5296757817268372,0.8604294657707214,-0.8895052075386047,0.9556970000267029,0.6629188656806946,0.4295697808265686,0.9736858010292053,0.655548095703125,0.12201863527297974,0.7442769408226013,0.5037954449653625,0.95253586769104,-0.6507166624069214,-0.9316278100013733,0.9791510701179504,-0.9972586035728455,0.9613031148910522,0.7501779794692993,-0.3806658983230591,0.6429785490036011,0.9557769298553467,-0.9428837895393372,-0.9948079586029053,0.7852989435195923,0.9657301306724548,0.7073014974594116,0.36872273683547974,0.8128010034561157,0.9659482836723328,0.9730120301246643,0.31900620460510254,-0.30290529131889343,0.9790952801704407,0.9357922673225403,-0.5550314784049988,-0.0054661668837070465,0.986777663230896,0.8249568343162537,0.5664296746253967,0.1000528484582901,-0.9464486837387085,-0.25471991300582886,0.6522327065467834,0.14152583479881287,0.9884141683578491,0.9860584735870361,0.6949271559715271,0.9901811480522156,0.9791203141212463,-0.23595543205738068,-0.982071042060852,0.6506688594818115,0.9895945191383362,-0.291781485080719,0.9714024662971497,0.9951602220535278,0.18783727288246155,-0.9460937976837158,0.4780191481113434,-0.2489192634820938,0.9437099099159241,0.11866225302219391,0.9941242933273315,-0.38088199496269226,0.9564487338066101,0.5542722344398499,0.42118069529533386,0.6628788113594055,0.8659593462944031,0.9937117695808411,0.9069075584411621,0.3981107473373413,-0.4134218096733093,0.9971914887428284,0.3459664583206177,0.9938657283782959,0.5891268253326416,0.9313738942146301,0.9268401861190796,0.9993563890457153,0.6227542161941528,0.8463947772979736,0.6584343910217285,0.8423123955726624,0.2978499233722687,0.8728678822517395,0.9963143467903137,0.9867526292800903],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"Full OV Copying Score"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"OV Copying Score"}},"legend":{"tracegroupgap":0},"title":{"text":"OV Copying Score for each head in GPT-2 Small"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('2be56e5c-d2b2-481c-94a5-bddccdf1e7cd');
+var gd = document.getElementById('2386d5b5-f55c-48a9-850d-e7c0f3dfdb0b');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -2178,7 +2170,7 @@ <h2>Generating Text<a class="headerlink" href="#Generating-Text" title="Permalin
 <div class="prompt empty docutils container">
 </div>
 <div class="output_area docutils container">
-<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "f1b89201b64941828ea8f7bb5ebac2ca"}</script></div>
+<script type="application/vnd.jupyter.widget-view+json">{"version_major": 2, "version_minor": 0, "model_id": "008a6d643cb74e9da094f214bdade5b2"}</script></div>
 </div>
 <div class="nboutput nblast docutils container">
 <div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[53]:
@@ -2336,9 +2328,9 @@ <h2>Loading Pre-Trained Checkpoints<a class="headerlink" href="#Loading-Pre-Trai
 <head><meta charset="utf-8" /></head>
 <body>
     <div>            <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG"></script><script type="text/javascript">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: "STIX-Web"}});}</script>                <script type="text/javascript">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>
-        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="2f4b0396-49d9-4cde-b98b-91ebcc4cd205" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("2f4b0396-49d9-4cde-b98b-91ebcc4cd205")) {                    Plotly.newPlot(                        "2f4b0396-49d9-4cde-b98b-91ebcc4cd205",                        [{"hovertemplate":"variable=0\u003cbr\u003eindex=%{x}\u003cbr\u003evalue=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"0","line":{"color":"#636efa","dash":"solid"},"marker":{"symbol":"circle"},"mode":"lines+markers","name":"0","orientation":"v","showlegend":true,"x":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162],"xaxis":"x","y":[262144,2621440,4718592,7077888,9175040,11272192,13631488,15728640,18087936,20185088,22282240,33292288,44302336,55312384,66322432,77332480,88342528,99352576,110362624,121372672,132382720,143392768,154402816,165412864,176422912,187432960,198443008,209453056,220463104,264503296,308281344,352321536,396361728,440401920,484442112,528482304,572522496,616300544,660340736,704380928,748421120,792461312,836501504,880279552,924319744,968359936,1012400128,1056440320,1100480512,1144520704,1188298752,1232338944,1276379136,1320419328,1364459520,1408499712,1452277760,1496317952,1540358144,1584398336,1628438528,1672478720,1716518912,1760296960,1804337152,1848377344,1892417536,1936457728,1980497920,2024275968,2068316160,2112356352,2156396544,2200436736,2420375552,2640314368,2860515328,3080454144,3300392960,3520331776,3740270592,3960471552,4180410368,4400349184,4620288000,4840488960,5060427776,5280366592,5500305408,5720506368,5940445184,6160384000,6380322816,6600523776,6820462592,7040401408,7260340224,7480279040,7700480000,7920418816,8140357632,8360296448,8580497408,8800436224,9020375040,9240313856,9460514816,9680453632,9900392448,10120331264,10340270080,10560471040,10780409856,11000348672,11220287488,11440488448,11660427264,11880366080,12100304896,12320505856,12540444672,12760383488,12980322304,13200523264,13420462080,13640400896,13860339712,14080278528,14300479488,14520418304,14740357120,14960295936,15180496896,15400435712,15620374528,15840313344,16060514304,16280453120,16500391936,16720330752,16940269568,17160470528,17380409344,17600348160,17820286976,18040487936,18260426752,18480365568,18700304384,18920505344,19140444160,19360382976,19580321792,19800522752,20020461568,20240400384,20460339200,20680278016,20900478976,21120417792,21340356608,21560295424,21780496384],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"index"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"value"},"type":"log"},"legend":{"title":{"text":"variable"},"tracegroupgap":0},"title":{"text":"Checkpoint Values for attn-only-2l (Log scale)"}},                        {"responsive": true}                    ).then(function(){
+        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="4602c771-fd44-40e0-a92b-9fc2cdaf024a" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("4602c771-fd44-40e0-a92b-9fc2cdaf024a")) {                    Plotly.newPlot(                        "4602c771-fd44-40e0-a92b-9fc2cdaf024a",                        [{"hovertemplate":"variable=0\u003cbr\u003eindex=%{x}\u003cbr\u003evalue=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"0","line":{"color":"#636efa","dash":"solid"},"marker":{"symbol":"circle"},"mode":"lines+markers","name":"0","orientation":"v","showlegend":true,"x":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162],"xaxis":"x","y":[262144,2621440,4718592,7077888,9175040,11272192,13631488,15728640,18087936,20185088,22282240,33292288,44302336,55312384,66322432,77332480,88342528,99352576,110362624,121372672,132382720,143392768,154402816,165412864,176422912,187432960,198443008,209453056,220463104,264503296,308281344,352321536,396361728,440401920,484442112,528482304,572522496,616300544,660340736,704380928,748421120,792461312,836501504,880279552,924319744,968359936,1012400128,1056440320,1100480512,1144520704,1188298752,1232338944,1276379136,1320419328,1364459520,1408499712,1452277760,1496317952,1540358144,1584398336,1628438528,1672478720,1716518912,1760296960,1804337152,1848377344,1892417536,1936457728,1980497920,2024275968,2068316160,2112356352,2156396544,2200436736,2420375552,2640314368,2860515328,3080454144,3300392960,3520331776,3740270592,3960471552,4180410368,4400349184,4620288000,4840488960,5060427776,5280366592,5500305408,5720506368,5940445184,6160384000,6380322816,6600523776,6820462592,7040401408,7260340224,7480279040,7700480000,7920418816,8140357632,8360296448,8580497408,8800436224,9020375040,9240313856,9460514816,9680453632,9900392448,10120331264,10340270080,10560471040,10780409856,11000348672,11220287488,11440488448,11660427264,11880366080,12100304896,12320505856,12540444672,12760383488,12980322304,13200523264,13420462080,13640400896,13860339712,14080278528,14300479488,14520418304,14740357120,14960295936,15180496896,15400435712,15620374528,15840313344,16060514304,16280453120,16500391936,16720330752,16940269568,17160470528,17380409344,17600348160,17820286976,18040487936,18260426752,18480365568,18700304384,18920505344,19140444160,19360382976,19580321792,19800522752,20020461568,20240400384,20460339200,20680278016,20900478976,21120417792,21340356608,21560295424,21780496384],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"index"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"value"},"type":"log"},"legend":{"title":{"text":"variable"},"tracegroupgap":0},"title":{"text":"Checkpoint Values for attn-only-2l (Log scale)"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('2f4b0396-49d9-4cde-b98b-91ebcc4cd205');
+var gd = document.getElementById('4602c771-fd44-40e0-a92b-9fc2cdaf024a');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -2372,9 +2364,9 @@ <h2>Loading Pre-Trained Checkpoints<a class="headerlink" href="#Loading-Pre-Trai
 <head><meta charset="utf-8" /></head>
 <body>
     <div>            <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG"></script><script type="text/javascript">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: "STIX-Web"}});}</script>                <script type="text/javascript">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>
-        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="f50d5618-52f9-42e8-a118-5e9a1a641854" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("f50d5618-52f9-42e8-a118-5e9a1a641854")) {                    Plotly.newPlot(                        "f50d5618-52f9-42e8-a118-5e9a1a641854",                        [{"hovertemplate":"variable=0\u003cbr\u003eindex=%{x}\u003cbr\u003evalue=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"0","line":{"color":"#636efa","dash":"solid"},"marker":{"symbol":"circle"},"mode":"lines+markers","name":"0","orientation":"v","showlegend":true,"x":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162],"xaxis":"x","y":[196608,3342336,6291456,9240576,12386304,15335424,18284544,21233664,24379392,27328512,30277632,45219840,60358656,75300864,90243072,105381888,120324096,135266304,150208512,165347328,180289536,195231744,210370560,225312768,240254976,255197184,270336000,285278208,300220416,360382464,420347904,480313344,540278784,600244224,660209664,720371712,780337152,840302592,900268032,960233472,1020198912,1080360960,1140326400,1200291840,1260257280,1320222720,1380384768,1440350208,1500315648,1560281088,1620246528,1680211968,1740374016,1800339456,1860304896,1920270336,1980235776,2040201216,2100363264,2160328704,2220294144,2280259584,2340225024,2400387072,2460352512,2520317952,2580283392,2640248832,2700214272,2760376320,2820341760,2880307200,2940272640,3000238080,3300261888,3600285696,3900309504,4200333312,4500357120,4800380928,5100208128,5400231936,5700255744,6000279552,6300303360,6600327168,6900350976,7200374784,7500201984,7800225792,8100249600,8400273408,8700297216,9000321024,9300344832,9600368640,9900392448,10200219648,10500243456,10800267264,11100291072,11400314880,11700338688,12000362496,12300386304,12600213504,12900237312,13200261120,13500284928,13800308736,14100332544,14400356352,14700380160,15000207360,15300231168,15600254976,15900278784,16200302592,16500326400,16800350208,17100374016,17400201216,17700225024,18000248832,18300272640,18600296448,18900320256,19200344064,19500367872,19800391680,20100218880,20400242688,20700266496,21000290304,21300314112,21600337920,21900361728,22200385536,22500212736,22800236544,23100260352,23400284160,23700307968,24000331776,24300355584,24600379392,24900206592,25200230400,25500254208,25800278016,26100301824,26400325632,26700349440,27000373248,27300200448,27600224256,27900248064,28200271872,28500295680,28800319488,29100343296,29400367104,29700390912],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"index"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"value"},"type":"log"},"legend":{"title":{"text":"variable"},"tracegroupgap":0},"title":{"text":"Checkpoint Values for solu-12l (Log scale)"}},                        {"responsive": true}                    ).then(function(){
+        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="49f0a47b-d076-4975-a8be-48982bf36ec5" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("49f0a47b-d076-4975-a8be-48982bf36ec5")) {                    Plotly.newPlot(                        "49f0a47b-d076-4975-a8be-48982bf36ec5",                        [{"hovertemplate":"variable=0\u003cbr\u003eindex=%{x}\u003cbr\u003evalue=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"0","line":{"color":"#636efa","dash":"solid"},"marker":{"symbol":"circle"},"mode":"lines+markers","name":"0","orientation":"v","showlegend":true,"x":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162],"xaxis":"x","y":[196608,3342336,6291456,9240576,12386304,15335424,18284544,21233664,24379392,27328512,30277632,45219840,60358656,75300864,90243072,105381888,120324096,135266304,150208512,165347328,180289536,195231744,210370560,225312768,240254976,255197184,270336000,285278208,300220416,360382464,420347904,480313344,540278784,600244224,660209664,720371712,780337152,840302592,900268032,960233472,1020198912,1080360960,1140326400,1200291840,1260257280,1320222720,1380384768,1440350208,1500315648,1560281088,1620246528,1680211968,1740374016,1800339456,1860304896,1920270336,1980235776,2040201216,2100363264,2160328704,2220294144,2280259584,2340225024,2400387072,2460352512,2520317952,2580283392,2640248832,2700214272,2760376320,2820341760,2880307200,2940272640,3000238080,3300261888,3600285696,3900309504,4200333312,4500357120,4800380928,5100208128,5400231936,5700255744,6000279552,6300303360,6600327168,6900350976,7200374784,7500201984,7800225792,8100249600,8400273408,8700297216,9000321024,9300344832,9600368640,9900392448,10200219648,10500243456,10800267264,11100291072,11400314880,11700338688,12000362496,12300386304,12600213504,12900237312,13200261120,13500284928,13800308736,14100332544,14400356352,14700380160,15000207360,15300231168,15600254976,15900278784,16200302592,16500326400,16800350208,17100374016,17400201216,17700225024,18000248832,18300272640,18600296448,18900320256,19200344064,19500367872,19800391680,20100218880,20400242688,20700266496,21000290304,21300314112,21600337920,21900361728,22200385536,22500212736,22800236544,23100260352,23400284160,23700307968,24000331776,24300355584,24600379392,24900206592,25200230400,25500254208,25800278016,26100301824,26400325632,26700349440,27000373248,27300200448,27600224256,27900248064,28200271872,28500295680,28800319488,29100343296,29400367104,29700390912],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"index"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"value"},"type":"log"},"legend":{"title":{"text":"variable"},"tracegroupgap":0},"title":{"text":"Checkpoint Values for solu-12l (Log scale)"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('f50d5618-52f9-42e8-a118-5e9a1a641854');
+var gd = document.getElementById('49f0a47b-d076-4975-a8be-48982bf36ec5');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -2408,9 +2400,9 @@ <h2>Loading Pre-Trained Checkpoints<a class="headerlink" href="#Loading-Pre-Trai
 <head><meta charset="utf-8" /></head>
 <body>
     <div>            <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG"></script><script type="text/javascript">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: "STIX-Web"}});}</script>                <script type="text/javascript">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>
-        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="9b48617d-0936-4432-a9e6-abbcbfa6eda0" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("9b48617d-0936-4432-a9e6-abbcbfa6eda0")) {                    Plotly.newPlot(                        "9b48617d-0936-4432-a9e6-abbcbfa6eda0",                        [{"hovertemplate":"variable=0\u003cbr\u003eindex=%{x}\u003cbr\u003evalue=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"0","line":{"color":"#636efa","dash":"solid"},"marker":{"symbol":"circle"},"mode":"lines+markers","name":"0","orientation":"v","showlegend":true,"x":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332,333,334,335,336,337,338,339,340,341,342,343,344,345,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365,366,367,368,369,370,371,372,373,374,375,376,377,378,379,380,381,382,383,384,385,386,387,388,389,390,391,392,393,394,395,396,397,398,399,400,401,402,403,404,405,406,407,408,409,410,411,412,413,414,415,416,417,418,419,420,421,422,423,424,425,426,427,428,429,430,431,432,433,434,435,436,437,438,439,440,441,442,443,444,445,446,447,448,449,450,451,452,453,454,455,456,457,458,459,460,461,462,463,464,465,466,467,468,469,470,471,472,473,474,475,476,477,478,479,480,481,482,483,484,485,486,487,488,489,490,491,492,493,494,495,496,497,498,499,500,501,502,503,504,505,506,507,508,509,510,511,512,513,514,515,516,517,518,519,520,521,522,523,524,525,526,527,528,529,530,531,532,533,534,535,536,537,538,539,540,541,542,543,544,545,546,547,548,549,550,551,552,553,554,555,556,557,558,559,560,561,562,563,564,565,566,567,568,569,570,571,572,573,574,575,576,577,578,579,580,581,582,583,584,585,586,587,588,589,590,591,592,593,594,595,596,597,598,599,600,601,602,603,604,605,606,607,608],"xaxis":"x","y":[0,10,20,30,40,50,60,70,80,90,100,150,200,250,300,350,400,450,500,550,600,650,700,750,800,850,900,950,1000,1050,1100,1150,1200,1250,1300,1350,1400,1450,1500,1550,1600,1650,1700,1750,1800,1850,1900,1950,2000,2100,2200,2300,2400,2500,2600,2700,2800,2900,3000,3100,3200,3300,3400,3500,3600,3700,3800,3900,4000,4100,4200,4300,4400,4500,4600,4700,4800,4900,5000,5100,5200,5300,5400,5500,5600,5700,5800,5900,6000,6100,6200,6300,6400,6500,6600,6700,6800,6900,7000,7100,7200,7300,7400,7500,7600,7700,7800,7900,8000,8100,8200,8300,8400,8500,8600,8700,8800,8900,9000,9100,9200,9300,9400,9500,9600,9700,9800,9900,10000,10100,10200,10300,10400,10500,10600,10700,10800,10900,11000,11100,11200,11300,11400,11500,11600,11700,11800,11900,12000,12100,12200,12300,12400,12500,12600,12700,12800,12900,13000,13100,13200,13300,13400,13500,13600,13700,13800,13900,14000,14100,14200,14300,14400,14500,14600,14700,14800,14900,15000,15100,15200,15300,15400,15500,15600,15700,15800,15900,16000,16100,16200,16300,16400,16500,16600,16700,16800,16900,17000,17100,17200,17300,17400,17500,17600,17700,17800,17900,18000,18100,18200,18300,18400,18500,18600,18700,18800,18900,19000,19100,19200,19300,19400,19500,19600,19700,19800,19900,20000,21000,22000,23000,24000,25000,26000,27000,28000,29000,30000,31000,32000,33000,34000,35000,36000,37000,38000,39000,40000,41000,42000,43000,44000,45000,46000,47000,48000,49000,50000,51000,52000,53000,54000,55000,56000,57000,58000,59000,60000,61000,62000,63000,64000,65000,66000,67000,68000,69000,70000,71000,72000,73000,74000,75000,76000,77000,78000,79000,80000,81000,82000,83000,84000,85000,86000,87000,88000,89000,90000,91000,92000,93000,94000,95000,96000,97000,98000,99000,100000,101000,102000,103000,104000,105000,106000,107000,108000,109000,110000,111000,112000,113000,114000,115000,116000,117000,118000,119000,120000,121000,122000,123000,124000,125000,126000,127000,128000,129000,130000,131000,132000,133000,134000,135000,136000,137000,138000,139000,140000,141000,142000,143000,144000,145000,146000,147000,148000,149000,150000,151000,152000,153000,154000,155000,156000,157000,158000,159000,160000,161000,162000,163000,164000,165000,166000,167000,168000,169000,170000,171000,172000,173000,174000,175000,176000,177000,178000,179000,180000,181000,182000,183000,184000,185000,186000,187000,188000,189000,190000,191000,192000,193000,194000,195000,196000,197000,198000,199000,200000,201000,202000,203000,204000,205000,206000,207000,208000,209000,210000,211000,212000,213000,214000,215000,216000,217000,218000,219000,220000,221000,222000,223000,224000,225000,226000,227000,228000,229000,230000,231000,232000,233000,234000,235000,236000,237000,238000,239000,240000,241000,242000,243000,244000,245000,246000,247000,248000,249000,250000,251000,252000,253000,254000,255000,256000,257000,258000,259000,260000,261000,262000,263000,264000,265000,266000,267000,268000,269000,270000,271000,272000,273000,274000,275000,276000,277000,278000,279000,280000,281000,282000,283000,284000,285000,286000,287000,288000,289000,290000,291000,292000,293000,294000,295000,296000,297000,298000,299000,300000,301000,302000,303000,304000,305000,306000,307000,308000,309000,310000,311000,312000,313000,314000,315000,316000,317000,318000,319000,320000,321000,322000,323000,324000,325000,326000,327000,328000,329000,330000,331000,332000,333000,334000,335000,336000,337000,338000,339000,340000,341000,342000,343000,344000,345000,346000,347000,348000,349000,350000,351000,352000,353000,354000,355000,356000,357000,358000,359000,360000,361000,362000,363000,364000,365000,366000,367000,368000,369000,370000,371000,372000,373000,374000,375000,376000,377000,378000,379000,380000,381000,382000,383000,384000,385000,386000,387000,388000,389000,390000,391000,392000,393000,394000,395000,396000,397000,398000,399000,400000],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"index"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"value"},"type":"log"},"legend":{"title":{"text":"variable"},"tracegroupgap":0},"title":{"text":"Checkpoint Values for stanford-gpt2-small-a (Log scale)"}},                        {"responsive": true}                    ).then(function(){
+        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="922fb711-b683-41df-bedc-350328d71e57" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("922fb711-b683-41df-bedc-350328d71e57")) {                    Plotly.newPlot(                        "922fb711-b683-41df-bedc-350328d71e57",                        [{"hovertemplate":"variable=0\u003cbr\u003eindex=%{x}\u003cbr\u003evalue=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"0","line":{"color":"#636efa","dash":"solid"},"marker":{"symbol":"circle"},"mode":"lines+markers","name":"0","orientation":"v","showlegend":true,"x":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332,333,334,335,336,337,338,339,340,341,342,343,344,345,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365,366,367,368,369,370,371,372,373,374,375,376,377,378,379,380,381,382,383,384,385,386,387,388,389,390,391,392,393,394,395,396,397,398,399,400,401,402,403,404,405,406,407,408,409,410,411,412,413,414,415,416,417,418,419,420,421,422,423,424,425,426,427,428,429,430,431,432,433,434,435,436,437,438,439,440,441,442,443,444,445,446,447,448,449,450,451,452,453,454,455,456,457,458,459,460,461,462,463,464,465,466,467,468,469,470,471,472,473,474,475,476,477,478,479,480,481,482,483,484,485,486,487,488,489,490,491,492,493,494,495,496,497,498,499,500,501,502,503,504,505,506,507,508,509,510,511,512,513,514,515,516,517,518,519,520,521,522,523,524,525,526,527,528,529,530,531,532,533,534,535,536,537,538,539,540,541,542,543,544,545,546,547,548,549,550,551,552,553,554,555,556,557,558,559,560,561,562,563,564,565,566,567,568,569,570,571,572,573,574,575,576,577,578,579,580,581,582,583,584,585,586,587,588,589,590,591,592,593,594,595,596,597,598,599,600,601,602,603,604,605,606,607,608],"xaxis":"x","y":[0,10,20,30,40,50,60,70,80,90,100,150,200,250,300,350,400,450,500,550,600,650,700,750,800,850,900,950,1000,1050,1100,1150,1200,1250,1300,1350,1400,1450,1500,1550,1600,1650,1700,1750,1800,1850,1900,1950,2000,2100,2200,2300,2400,2500,2600,2700,2800,2900,3000,3100,3200,3300,3400,3500,3600,3700,3800,3900,4000,4100,4200,4300,4400,4500,4600,4700,4800,4900,5000,5100,5200,5300,5400,5500,5600,5700,5800,5900,6000,6100,6200,6300,6400,6500,6600,6700,6800,6900,7000,7100,7200,7300,7400,7500,7600,7700,7800,7900,8000,8100,8200,8300,8400,8500,8600,8700,8800,8900,9000,9100,9200,9300,9400,9500,9600,9700,9800,9900,10000,10100,10200,10300,10400,10500,10600,10700,10800,10900,11000,11100,11200,11300,11400,11500,11600,11700,11800,11900,12000,12100,12200,12300,12400,12500,12600,12700,12800,12900,13000,13100,13200,13300,13400,13500,13600,13700,13800,13900,14000,14100,14200,14300,14400,14500,14600,14700,14800,14900,15000,15100,15200,15300,15400,15500,15600,15700,15800,15900,16000,16100,16200,16300,16400,16500,16600,16700,16800,16900,17000,17100,17200,17300,17400,17500,17600,17700,17800,17900,18000,18100,18200,18300,18400,18500,18600,18700,18800,18900,19000,19100,19200,19300,19400,19500,19600,19700,19800,19900,20000,21000,22000,23000,24000,25000,26000,27000,28000,29000,30000,31000,32000,33000,34000,35000,36000,37000,38000,39000,40000,41000,42000,43000,44000,45000,46000,47000,48000,49000,50000,51000,52000,53000,54000,55000,56000,57000,58000,59000,60000,61000,62000,63000,64000,65000,66000,67000,68000,69000,70000,71000,72000,73000,74000,75000,76000,77000,78000,79000,80000,81000,82000,83000,84000,85000,86000,87000,88000,89000,90000,91000,92000,93000,94000,95000,96000,97000,98000,99000,100000,101000,102000,103000,104000,105000,106000,107000,108000,109000,110000,111000,112000,113000,114000,115000,116000,117000,118000,119000,120000,121000,122000,123000,124000,125000,126000,127000,128000,129000,130000,131000,132000,133000,134000,135000,136000,137000,138000,139000,140000,141000,142000,143000,144000,145000,146000,147000,148000,149000,150000,151000,152000,153000,154000,155000,156000,157000,158000,159000,160000,161000,162000,163000,164000,165000,166000,167000,168000,169000,170000,171000,172000,173000,174000,175000,176000,177000,178000,179000,180000,181000,182000,183000,184000,185000,186000,187000,188000,189000,190000,191000,192000,193000,194000,195000,196000,197000,198000,199000,200000,201000,202000,203000,204000,205000,206000,207000,208000,209000,210000,211000,212000,213000,214000,215000,216000,217000,218000,219000,220000,221000,222000,223000,224000,225000,226000,227000,228000,229000,230000,231000,232000,233000,234000,235000,236000,237000,238000,239000,240000,241000,242000,243000,244000,245000,246000,247000,248000,249000,250000,251000,252000,253000,254000,255000,256000,257000,258000,259000,260000,261000,262000,263000,264000,265000,266000,267000,268000,269000,270000,271000,272000,273000,274000,275000,276000,277000,278000,279000,280000,281000,282000,283000,284000,285000,286000,287000,288000,289000,290000,291000,292000,293000,294000,295000,296000,297000,298000,299000,300000,301000,302000,303000,304000,305000,306000,307000,308000,309000,310000,311000,312000,313000,314000,315000,316000,317000,318000,319000,320000,321000,322000,323000,324000,325000,326000,327000,328000,329000,330000,331000,332000,333000,334000,335000,336000,337000,338000,339000,340000,341000,342000,343000,344000,345000,346000,347000,348000,349000,350000,351000,352000,353000,354000,355000,356000,357000,358000,359000,360000,361000,362000,363000,364000,365000,366000,367000,368000,369000,370000,371000,372000,373000,374000,375000,376000,377000,378000,379000,380000,381000,382000,383000,384000,385000,386000,387000,388000,389000,390000,391000,392000,393000,394000,395000,396000,397000,398000,399000,400000],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"index"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"value"},"type":"log"},"legend":{"title":{"text":"variable"},"tracegroupgap":0},"title":{"text":"Checkpoint Values for stanford-gpt2-small-a (Log scale)"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('9b48617d-0936-4432-a9e6-abbcbfa6eda0');
+var gd = document.getElementById('922fb711-b683-41df-bedc-350328d71e57');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -2444,9 +2436,9 @@ <h2>Loading Pre-Trained Checkpoints<a class="headerlink" href="#Loading-Pre-Trai
 <head><meta charset="utf-8" /></head>
 <body>
     <div>            <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG"></script><script type="text/javascript">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: "STIX-Web"}});}</script>                <script type="text/javascript">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>
-        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="e382771d-4fc5-4455-9679-fc20a7ee3b0c" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("e382771d-4fc5-4455-9679-fc20a7ee3b0c")) {                    Plotly.newPlot(                        "e382771d-4fc5-4455-9679-fc20a7ee3b0c",                        [{"hovertemplate":"variable=0\u003cbr\u003eindex=%{x}\u003cbr\u003evalue=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"0","line":{"color":"#636efa","dash":"solid"},"marker":{"symbol":"circle"},"mode":"lines+markers","name":"0","orientation":"v","showlegend":true,"x":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49],"xaxis":"x","y":[832,1664,2496,3328,4160,4992,5824,6656,7488,8320,9152,9984,10816,11648,12480,13312,14144,14976,15808,16640,17472,18304,19136,19968,20800,21632,22464,23296,24128,24960,25792,26624,27456,28288,29120,29952,30784,31616,32448,33280,34112,34944,35776,36608,37440,38272,39104,39936,40768,41600],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"index"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"value"}},"legend":{"title":{"text":"variable"},"tracegroupgap":0},"title":{"text":"Checkpoint Values for solu-1l-pile (Linear scale)"}},                        {"responsive": true}                    ).then(function(){
+        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="c9c2aa36-ee52-483a-bc6c-917a627d3f7e" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("c9c2aa36-ee52-483a-bc6c-917a627d3f7e")) {                    Plotly.newPlot(                        "c9c2aa36-ee52-483a-bc6c-917a627d3f7e",                        [{"hovertemplate":"variable=0\u003cbr\u003eindex=%{x}\u003cbr\u003evalue=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"0","line":{"color":"#636efa","dash":"solid"},"marker":{"symbol":"circle"},"mode":"lines+markers","name":"0","orientation":"v","showlegend":true,"x":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49],"xaxis":"x","y":[832,1664,2496,3328,4160,4992,5824,6656,7488,8320,9152,9984,10816,11648,12480,13312,14144,14976,15808,16640,17472,18304,19136,19968,20800,21632,22464,23296,24128,24960,25792,26624,27456,28288,29120,29952,30784,31616,32448,33280,34112,34944,35776,36608,37440,38272,39104,39936,40768,41600],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"index"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"value"}},"legend":{"title":{"text":"variable"},"tracegroupgap":0},"title":{"text":"Checkpoint Values for solu-1l-pile (Linear scale)"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('e382771d-4fc5-4455-9679-fc20a7ee3b0c');
+var gd = document.getElementById('c9c2aa36-ee52-483a-bc6c-917a627d3f7e');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -2480,9 +2472,9 @@ <h2>Loading Pre-Trained Checkpoints<a class="headerlink" href="#Loading-Pre-Trai
 <head><meta charset="utf-8" /></head>
 <body>
     <div>            <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG"></script><script type="text/javascript">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: "STIX-Web"}});}</script>                <script type="text/javascript">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>
-        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="c97a30e3-81bb-4a39-8684-28490def05f7" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("c97a30e3-81bb-4a39-8684-28490def05f7")) {                    Plotly.newPlot(                        "c97a30e3-81bb-4a39-8684-28490def05f7",                        [{"hovertemplate":"variable=0\u003cbr\u003eindex=%{x}\u003cbr\u003evalue=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"0","line":{"color":"#636efa","dash":"solid"},"marker":{"symbol":"circle"},"mode":"lines+markers","name":"0","orientation":"v","showlegend":true,"x":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99],"xaxis":"x","y":[326,652,978,1304,1630,1956,2282,2608,2934,3260,3586,3912,4238,4564,4890,5216,5542,5868,6194,6520,6846,7172,7498,7824,8150,8476,8802,9128,9454,9780,10106,10432,10758,11084,11410,11736,12062,12388,12714,13040,13366,13692,14018,14344,14670,14996,15322,15648,15974,16300,16626,16952,17278,17604,17930,18256,18582,18908,19234,19560,19886,20212,20538,20864,21190,21516,21842,22168,22494,22820,23146,23472,23798,24124,24450,24776,25102,25428,25754,26080,26406,26732,27058,27384,27710,28036,28362,28688,29014,29340,29666,29992,30318,30644,30970,31296,31622,31948,32274,32600],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"index"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"value"}},"legend":{"title":{"text":"variable"},"tracegroupgap":0},"title":{"text":"Checkpoint Values for solu-6l-pile (Linear scale)"}},                        {"responsive": true}                    ).then(function(){
+        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="866c87d0-3047-4894-870f-525c60a8f2ad" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("866c87d0-3047-4894-870f-525c60a8f2ad")) {                    Plotly.newPlot(                        "866c87d0-3047-4894-870f-525c60a8f2ad",                        [{"hovertemplate":"variable=0\u003cbr\u003eindex=%{x}\u003cbr\u003evalue=%{y}\u003cextra\u003e\u003c\u002fextra\u003e","legendgroup":"0","line":{"color":"#636efa","dash":"solid"},"marker":{"symbol":"circle"},"mode":"lines+markers","name":"0","orientation":"v","showlegend":true,"x":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99],"xaxis":"x","y":[326,652,978,1304,1630,1956,2282,2608,2934,3260,3586,3912,4238,4564,4890,5216,5542,5868,6194,6520,6846,7172,7498,7824,8150,8476,8802,9128,9454,9780,10106,10432,10758,11084,11410,11736,12062,12388,12714,13040,13366,13692,14018,14344,14670,14996,15322,15648,15974,16300,16626,16952,17278,17604,17930,18256,18582,18908,19234,19560,19886,20212,20538,20864,21190,21516,21842,22168,22494,22820,23146,23472,23798,24124,24450,24776,25102,25428,25754,26080,26406,26732,27058,27384,27710,28036,28362,28688,29014,29340,29666,29992,30318,30644,30970,31296,31622,31948,32274,32600],"yaxis":"y","type":"scatter"}],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"index"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"value"}},"legend":{"title":{"text":"variable"},"tracegroupgap":0},"title":{"text":"Checkpoint Values for solu-6l-pile (Linear scale)"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('c97a30e3-81bb-4a39-8684-28490def05f7');
+var gd = document.getElementById('866c87d0-3047-4894-870f-525c60a8f2ad');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -2567,9 +2559,9 @@ <h3>Example: Induction Head Phase Transition<a class="headerlink" href="#Example
 <head><meta charset="utf-8" /></head>
 <body>
     <div>            <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG"></script><script type="text/javascript">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: "STIX-Web"}});}</script>                <script type="text/javascript">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>
-        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="a9eebf5f-2f5e-447e-90de-cbb3a18ef60b" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("a9eebf5f-2f5e-447e-90de-cbb3a18ef60b")) {                    Plotly.newPlot(                        "a9eebf5f-2f5e-447e-90de-cbb3a18ef60b",                        [],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"value"},"type":"log"},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"index"}},"legend":{"tracegroupgap":0},"title":{"text":"Induction Loss over training: solu-2l"}},                        {"responsive": true}                    ).then(function(){
+        <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.30.0.min.js"></script>                <div id="b9f31b41-2055-4494-8578-defbee4966cc" class="plotly-graph-div" style="height:525px; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("b9f31b41-2055-4494-8578-defbee4966cc")) {                    Plotly.newPlot(                        "b9f31b41-2055-4494-8578-defbee4966cc",                        [],                        {"template":{"data":{"histogram2dcontour":[{"type":"histogram2dcontour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"choropleth":[{"type":"choropleth","colorbar":{"outlinewidth":0,"ticks":""}}],"histogram2d":[{"type":"histogram2d","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmap":[{"type":"heatmap","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"heatmapgl":[{"type":"heatmapgl","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"contourcarpet":[{"type":"contourcarpet","colorbar":{"outlinewidth":0,"ticks":""}}],"contour":[{"type":"contour","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"surface":[{"type":"surface","colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]}],"mesh3d":[{"type":"mesh3d","colorbar":{"outlinewidth":0,"ticks":""}}],"scatter":[{"fillpattern":{"fillmode":"overlay","size":10,"solidity":0.2},"type":"scatter"}],"parcoords":[{"type":"parcoords","line":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolargl":[{"type":"scatterpolargl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"scattergeo":[{"type":"scattergeo","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterpolar":[{"type":"scatterpolar","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"scattergl":[{"type":"scattergl","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatter3d":[{"type":"scatter3d","line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattermapbox":[{"type":"scattermapbox","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scatterternary":[{"type":"scatterternary","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"scattercarpet":[{"type":"scattercarpet","marker":{"colorbar":{"outlinewidth":0,"ticks":""}}}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"pie":[{"automargin":true,"type":"pie"}]},"layout":{"autotypenumbers":"strict","colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"hovermode":"closest","hoverlabel":{"align":"left"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"bgcolor":"#E5ECF6","angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"ternary":{"bgcolor":"#E5ECF6","aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]]},"xaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"yaxis":{"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","automargin":true,"zerolinewidth":2},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white","gridwidth":2}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"geo":{"bgcolor":"white","landcolor":"#E5ECF6","subunitcolor":"white","showland":true,"showlakes":true,"lakecolor":"white"},"title":{"x":0.05},"mapbox":{"style":"light"}}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"value"},"type":"log"},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"index"}},"legend":{"tracegroupgap":0},"title":{"text":"Induction Loss over training: solu-2l"}},                        {"responsive": true}                    ).then(function(){
 
-var gd = document.getElementById('a9eebf5f-2f5e-447e-90de-cbb3a18ef60b');
+var gd = document.getElementById('b9f31b41-2055-4494-8578-defbee4966cc');
 var x = new MutationObserver(function (mutations, observer) {{
         var display = window.getComputedStyle(gd).display;
         if (!display || display === 'none') {{
@@ -2596,7 +2588,7 @@ <h3>Example: Induction Head Phase Transition<a class="headerlink" href="#Example
 </html></div>
 </div>
 <script type="application/vnd.jupyter.widget-state+json">
-{"state": {"8cf7b30b11d248e8b78f1080528d4668": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "4ac62c1c592f4a07b3845c7bb8f780e0": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "bc171be6fc9e4023a280df273ea78c7c": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_8cf7b30b11d248e8b78f1080528d4668", "max": 12.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_4ac62c1c592f4a07b3845c7bb8f780e0", "tabbable": null, "tooltip": null, "value": 12.0}}, "2ccf74a736494e378f51375790cdc75b": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "ea105ad53b6b44f9be96f599c6d06ec6": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "96fa2acab8fe4ebeb2e3c5faee788754": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_2ccf74a736494e378f51375790cdc75b", "placeholder": "\u200b", "style": "IPY_MODEL_ea105ad53b6b44f9be96f599c6d06ec6", "tabbable": null, "tooltip": null, "value": "100%"}}, "c847691a3c11405db8eb8fbc5538f3a7": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "da4b8443fbbd4207ab5bab911bc774e4": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "8852cffea1d647bc969df746c022c33e": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_c847691a3c11405db8eb8fbc5538f3a7", "placeholder": "\u200b", "style": "IPY_MODEL_da4b8443fbbd4207ab5bab911bc774e4", "tabbable": null, "tooltip": null, "value": "\u200712/12\u2007[00:20&lt;00:00,\u2007\u20071.70s/it]"}}, "a3debcca1d614acfab710a737555ee78": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "3829b426693a411babc613fc1d68b1c8": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_96fa2acab8fe4ebeb2e3c5faee788754", "IPY_MODEL_bc171be6fc9e4023a280df273ea78c7c", "IPY_MODEL_8852cffea1d647bc969df746c022c33e"], "layout": "IPY_MODEL_a3debcca1d614acfab710a737555ee78", "tabbable": null, "tooltip": null}}, "01811a7afc2b407fa4c58caf11506bf7": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "fd4d1fae71734f6e814c4fd7fa8819a0": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "5315a350c488485bb93cf6d1e571e6fb": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_01811a7afc2b407fa4c58caf11506bf7", "max": 352824413.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_fd4d1fae71734f6e814c4fd7fa8819a0", "tabbable": null, "tooltip": null, "value": 352824413.0}}, "0c173e347b6041b0875f9d2c303b3492": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "e4ceba62766246a3880ce34c46382396": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "9326ff21288f448f89edcf15a115fee9": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_0c173e347b6041b0875f9d2c303b3492", "placeholder": "\u200b", "style": "IPY_MODEL_e4ceba62766246a3880ce34c46382396", "tabbable": null, "tooltip": null, "value": "model.safetensors:\u2007100%"}}, "bc777df5093d45d4a5dc88af7705a3f7": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "32687fd245d5475c8dfa47143f4669a9": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "f957f6783dde4a2e8798c7f61d6e60d7": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_bc777df5093d45d4a5dc88af7705a3f7", "placeholder": "\u200b", "style": "IPY_MODEL_32687fd245d5475c8dfa47143f4669a9", "tabbable": null, "tooltip": null, "value": "\u2007353M/353M\u2007[00:02&lt;00:00,\u2007173MB/s]"}}, "c01db078359843f9b29c1ed34665a130": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "43da9025ada5462c9ceacafb0040d9e5": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_9326ff21288f448f89edcf15a115fee9", "IPY_MODEL_5315a350c488485bb93cf6d1e571e6fb", "IPY_MODEL_f957f6783dde4a2e8798c7f61d6e60d7"], "layout": "IPY_MODEL_c01db078359843f9b29c1ed34665a130", "tabbable": null, "tooltip": null}}, "9ab9621975614f0b81b69331fd0331f1": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "055a5114eef14f54946752c86917371d": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "e46ead82ee504133960b3c99b8f7e0d7": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_9ab9621975614f0b81b69331fd0331f1", "max": 124.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_055a5114eef14f54946752c86917371d", "tabbable": null, "tooltip": null, "value": 124.0}}, "3b2a4f2b968b4b788fa33b7258842629": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "5e773a3416684f3c966170693f88694a": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "4e1981eef4314877bea01005cb920f32": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_3b2a4f2b968b4b788fa33b7258842629", "placeholder": "\u200b", "style": "IPY_MODEL_5e773a3416684f3c966170693f88694a", "tabbable": null, "tooltip": null, "value": "generation_config.json:\u2007100%"}}, "f59255f1494b47948b5460064e7c3abc": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "44380ddd83d840b585e07399013c9c29": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "115505dc47284f0cb6169d331509ef32": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_f59255f1494b47948b5460064e7c3abc", "placeholder": "\u200b", "style": "IPY_MODEL_44380ddd83d840b585e07399013c9c29", "tabbable": null, "tooltip": null, "value": "\u2007124/124\u2007[00:00&lt;00:00,\u200722.2kB/s]"}}, "8c9c505dabf74e98a237f21c65b95367": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "b9efbbee427945918852a8f8351f3658": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_4e1981eef4314877bea01005cb920f32", "IPY_MODEL_e46ead82ee504133960b3c99b8f7e0d7", "IPY_MODEL_115505dc47284f0cb6169d331509ef32"], "layout": "IPY_MODEL_8c9c505dabf74e98a237f21c65b95367", "tabbable": null, "tooltip": null}}, "322463a77fcc4ff4afe1f15b79f65fca": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "e2f3b2b4710947b19dbb733d4d21d91b": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "ae6c5587383a4acb854b423ba322ae80": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_322463a77fcc4ff4afe1f15b79f65fca", "max": 26.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_e2f3b2b4710947b19dbb733d4d21d91b", "tabbable": null, "tooltip": null, "value": 26.0}}, "ba1b85b866e34cc1bd55e06fe523f82c": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "1573df0e781a416db610e58ef2fd2d0a": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "e557528f972e45e1b5ece9fd2b6f0ee6": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_ba1b85b866e34cc1bd55e06fe523f82c", "placeholder": "\u200b", "style": "IPY_MODEL_1573df0e781a416db610e58ef2fd2d0a", "tabbable": null, "tooltip": null, "value": "tokenizer_config.json:\u2007100%"}}, "ef20ddb2573249aa95c09444c24ca53c": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "694ab2d491454882a879562487c32abe": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "dfabbde516be47c5b62569906c3e982f": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_ef20ddb2573249aa95c09444c24ca53c", "placeholder": "\u200b", "style": "IPY_MODEL_694ab2d491454882a879562487c32abe", "tabbable": null, "tooltip": null, "value": "\u200726.0/26.0\u2007[00:00&lt;00:00,\u20074.76kB/s]"}}, "e220b630f756444aa598cd8265a6fc04": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "cad219cb55d34a84a8bd505e58157885": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_e557528f972e45e1b5ece9fd2b6f0ee6", "IPY_MODEL_ae6c5587383a4acb854b423ba322ae80", "IPY_MODEL_dfabbde516be47c5b62569906c3e982f"], "layout": "IPY_MODEL_e220b630f756444aa598cd8265a6fc04", "tabbable": null, "tooltip": null}}, "165200e8e98047229e09b70f281a34d8": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "028e71eaf7884a1880a18fbdb6a38252": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "250c17fa1cd749caa3494c908881efad": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_165200e8e98047229e09b70f281a34d8", "max": 1042301.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_028e71eaf7884a1880a18fbdb6a38252", "tabbable": null, "tooltip": null, "value": 1042301.0}}, "9cf4f7f88d94419fa1982c343fb3a1f1": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "096cc4aaadb64283bd8a6660dfe35bb9": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "49a5e94825ea4a69a197ce9ec10f51d0": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_9cf4f7f88d94419fa1982c343fb3a1f1", "placeholder": "\u200b", "style": "IPY_MODEL_096cc4aaadb64283bd8a6660dfe35bb9", "tabbable": null, "tooltip": null, "value": "vocab.json:\u2007100%"}}, "13940d7c73eb495aa70fefdeb6d7ec55": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "1bb496a0b62e46b7a6a38f64c6f62b3a": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "d83569bf74c141cfbdf849a272ad616b": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_13940d7c73eb495aa70fefdeb6d7ec55", "placeholder": "\u200b", "style": "IPY_MODEL_1bb496a0b62e46b7a6a38f64c6f62b3a", "tabbable": null, "tooltip": null, "value": "\u20071.04M/1.04M\u2007[00:00&lt;00:00,\u20073.89MB/s]"}}, "3e9e97a1b1f2449792386a93d3ef9da9": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "e60d70eb78324541b8f9e1d0766491c8": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_49a5e94825ea4a69a197ce9ec10f51d0", "IPY_MODEL_250c17fa1cd749caa3494c908881efad", "IPY_MODEL_d83569bf74c141cfbdf849a272ad616b"], "layout": "IPY_MODEL_3e9e97a1b1f2449792386a93d3ef9da9", "tabbable": null, "tooltip": null}}, "cee2581a67e94c1dafa02bb2a5f265ef": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "7d4541a6431b40ea8fd8247a31f9dadb": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "5244d573d015449085961de4b348bd21": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_cee2581a67e94c1dafa02bb2a5f265ef", "max": 456318.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_7d4541a6431b40ea8fd8247a31f9dadb", "tabbable": null, "tooltip": null, "value": 456318.0}}, "c2e2458e31524312bdc7ea8c65228182": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "c68354899e65466ca137578d798e2cd0": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "d243f9a93c5a48f2b7a26a83d8f8d5d8": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_c2e2458e31524312bdc7ea8c65228182", "placeholder": "\u200b", "style": "IPY_MODEL_c68354899e65466ca137578d798e2cd0", "tabbable": null, "tooltip": null, "value": "merges.txt:\u2007100%"}}, "477ab2bf7b384716a706efe251c0af4c": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "855aff9f5430465ab4af6a62c05168d4": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "b2882f0d52c848a0a48b546d96b933a6": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_477ab2bf7b384716a706efe251c0af4c", "placeholder": "\u200b", "style": "IPY_MODEL_855aff9f5430465ab4af6a62c05168d4", "tabbable": null, "tooltip": null, "value": "\u2007456k/456k\u2007[00:00&lt;00:00,\u20077.52MB/s]"}}, "0d01a56cb3dc4a3eab1a902fb310d8a2": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "40e09b2a289644d4b6031bd159605d65": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_d243f9a93c5a48f2b7a26a83d8f8d5d8", "IPY_MODEL_5244d573d015449085961de4b348bd21", "IPY_MODEL_b2882f0d52c848a0a48b546d96b933a6"], "layout": "IPY_MODEL_0d01a56cb3dc4a3eab1a902fb310d8a2", "tabbable": null, "tooltip": null}}, "e5b32706429e41539f524d47ea4f705a": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "9685337c8c4440f9b6d04307abbf9e0e": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "194a762a1d14455bad5c75294d79ef81": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_e5b32706429e41539f524d47ea4f705a", "max": 1355256.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_9685337c8c4440f9b6d04307abbf9e0e", "tabbable": null, "tooltip": null, "value": 1355256.0}}, "a1ffad5838934b0c895ba8c39cd34403": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "9a04f47585af4b21b30d2de91d685789": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "9c8e5ec575d84ea6a5c96aa4c749570e": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_a1ffad5838934b0c895ba8c39cd34403", "placeholder": "\u200b", "style": "IPY_MODEL_9a04f47585af4b21b30d2de91d685789", "tabbable": null, "tooltip": null, "value": "tokenizer.json:\u2007100%"}}, "840d3ecc67a84786a2508c01e23310ca": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "0411b4e36025480fb3916c53af875780": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "87b5f4d77add4c7ca78fa2e7e6e139aa": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_840d3ecc67a84786a2508c01e23310ca", "placeholder": "\u200b", "style": "IPY_MODEL_0411b4e36025480fb3916c53af875780", "tabbable": null, "tooltip": null, "value": "\u20071.36M/1.36M\u2007[00:00&lt;00:00,\u20076.79MB/s]"}}, "da0f3a0ffd2945939356d855658d90a3": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "8bccf23ea8614f7790636e7dd69b28b0": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_9c8e5ec575d84ea6a5c96aa4c749570e", "IPY_MODEL_194a762a1d14455bad5c75294d79ef81", "IPY_MODEL_87b5f4d77add4c7ca78fa2e7e6e139aa"], "layout": "IPY_MODEL_da0f3a0ffd2945939356d855658d90a3", "tabbable": null, "tooltip": null}}, "4be5583eb28a498089221a44849a7403": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "9559f304126340de8391a00ffc2af8d2": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "839b0fe8fd1d4914a256d60dc2a52b91": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_4be5583eb28a498089221a44849a7403", "max": 50.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_9559f304126340de8391a00ffc2af8d2", "tabbable": null, "tooltip": null, "value": 50.0}}, "6c5f0b95033347c6bb99585ee8623a82": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "7660b779de4941388ef8e0086abfbf57": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "232a3ab348284b429955adb9ce9b6a92": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_6c5f0b95033347c6bb99585ee8623a82", "placeholder": "\u200b", "style": "IPY_MODEL_7660b779de4941388ef8e0086abfbf57", "tabbable": null, "tooltip": null, "value": "100%"}}, "15aa6083796b453b8a9f32e1db888752": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "6cd874c59e2f48279e4d571fdd07a1b9": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "83389a919e9c4569afc116065eb7c4f0": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_15aa6083796b453b8a9f32e1db888752", "placeholder": "\u200b", "style": "IPY_MODEL_6cd874c59e2f48279e4d571fdd07a1b9", "tabbable": null, "tooltip": null, "value": "\u200750/50\u2007[00:02&lt;00:00,\u200722.25it/s]"}}, "2fab1eae450f479eb2db64cd3f12ed02": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "f1b89201b64941828ea8f7bb5ebac2ca": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_232a3ab348284b429955adb9ce9b6a92", "IPY_MODEL_839b0fe8fd1d4914a256d60dc2a52b91", "IPY_MODEL_83389a919e9c4569afc116065eb7c4f0"], "layout": "IPY_MODEL_2fab1eae450f479eb2db64cd3f12ed02", "tabbable": null, "tooltip": null}}}, "version_major": 2, "version_minor": 0}
+{"state": {"9694b20aca654e85874e76f287a305e7": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "b8094fe24f244d488a00415198028cbb": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "cddcf25b67ea4b8f8e31b2392f4f1a4a": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_9694b20aca654e85874e76f287a305e7", "max": 12.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_b8094fe24f244d488a00415198028cbb", "tabbable": null, "tooltip": null, "value": 12.0}}, "5d3afc1d48044c3baea5bf77f35c2752": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "ca6db46f215a46c7940fd3473e34187b": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "874725d00c554c14a10da4f89496cd4b": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_5d3afc1d48044c3baea5bf77f35c2752", "placeholder": "\u200b", "style": "IPY_MODEL_ca6db46f215a46c7940fd3473e34187b", "tabbable": null, "tooltip": null, "value": "100%"}}, "10567f086d8f46f5990a99a3731324fa": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "f0eb65695a344d84b0e6f62f40160520": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "4985934faaea4564bff9cdf370f1d3dc": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_10567f086d8f46f5990a99a3731324fa", "placeholder": "\u200b", "style": "IPY_MODEL_f0eb65695a344d84b0e6f62f40160520", "tabbable": null, "tooltip": null, "value": "\u200712/12\u2007[00:20&lt;00:00,\u2007\u20071.69s/it]"}}, "4d9487d84e154880b53742b321132162": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "8de918cf34204469a9a6abae1699ad36": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_874725d00c554c14a10da4f89496cd4b", "IPY_MODEL_cddcf25b67ea4b8f8e31b2392f4f1a4a", "IPY_MODEL_4985934faaea4564bff9cdf370f1d3dc"], "layout": "IPY_MODEL_4d9487d84e154880b53742b321132162", "tabbable": null, "tooltip": null}}, "f12fcdf158564aa880d46a54eea242dc": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "972d1265bbb44473922b191ca7aa89ab": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "5f23d06f2b22447586f73d329fd6f1b7": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_f12fcdf158564aa880d46a54eea242dc", "max": 352824413.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_972d1265bbb44473922b191ca7aa89ab", "tabbable": null, "tooltip": null, "value": 352824413.0}}, "8aa14118988e4695b82ec46fe96aa5e9": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "fb6a6a3f02be468c877ae97385dc5f12": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "0516cd1f75a04889997cb7327a9fdf3c": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_8aa14118988e4695b82ec46fe96aa5e9", "placeholder": "\u200b", "style": "IPY_MODEL_fb6a6a3f02be468c877ae97385dc5f12", "tabbable": null, "tooltip": null, "value": "model.safetensors:\u2007100%"}}, "74f9c61a4e3f413b90ed8f2547071ebf": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "8153259059cb4050b766ada3a5eac8a2": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "f4eaf5d519f749e4b7ecef7567833681": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_74f9c61a4e3f413b90ed8f2547071ebf", "placeholder": "\u200b", "style": "IPY_MODEL_8153259059cb4050b766ada3a5eac8a2", "tabbable": null, "tooltip": null, "value": "\u2007353M/353M\u2007[00:01&lt;00:00,\u2007298MB/s]"}}, "540faf238641448f809e912b89f0bb54": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "70c3c51f506a4de4974cf045dc40e642": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_0516cd1f75a04889997cb7327a9fdf3c", "IPY_MODEL_5f23d06f2b22447586f73d329fd6f1b7", "IPY_MODEL_f4eaf5d519f749e4b7ecef7567833681"], "layout": "IPY_MODEL_540faf238641448f809e912b89f0bb54", "tabbable": null, "tooltip": null}}, "447d8c1ba33641338c516f181b3ba8c7": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "650b710c29684ecdb05b2f11095c67eb": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "c7f63f2ff6ce48cea1c9ef6a477116a4": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_447d8c1ba33641338c516f181b3ba8c7", "max": 124.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_650b710c29684ecdb05b2f11095c67eb", "tabbable": null, "tooltip": null, "value": 124.0}}, "cbf2fd0392da454c9f0e289065e20bd5": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "b0731804db16433284acb42a5bf074a2": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "dfd44a1536764bb291c34f3d7ee5e8f8": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_cbf2fd0392da454c9f0e289065e20bd5", "placeholder": "\u200b", "style": "IPY_MODEL_b0731804db16433284acb42a5bf074a2", "tabbable": null, "tooltip": null, "value": "generation_config.json:\u2007100%"}}, "688ceb57a6834513a95e50899f15069f": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "28b414de4050442f9e8dcf15d5cd7039": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "a07aaaca4898479bb0086daf235ab228": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_688ceb57a6834513a95e50899f15069f", "placeholder": "\u200b", "style": "IPY_MODEL_28b414de4050442f9e8dcf15d5cd7039", "tabbable": null, "tooltip": null, "value": "\u2007124/124\u2007[00:00&lt;00:00,\u200721.9kB/s]"}}, "3d5179253d12488e898aa47bf606dfe0": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "c2024b930e0f46828f9f1935807c38fd": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_dfd44a1536764bb291c34f3d7ee5e8f8", "IPY_MODEL_c7f63f2ff6ce48cea1c9ef6a477116a4", "IPY_MODEL_a07aaaca4898479bb0086daf235ab228"], "layout": "IPY_MODEL_3d5179253d12488e898aa47bf606dfe0", "tabbable": null, "tooltip": null}}, "c2f46efa04404481889ef8a7867da37d": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "43b2e32b46f24994a24aeb9e5b160170": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "1588536871044d748718349a6c4ee375": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_c2f46efa04404481889ef8a7867da37d", "max": 26.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_43b2e32b46f24994a24aeb9e5b160170", "tabbable": null, "tooltip": null, "value": 26.0}}, "8f8518460bd1486f980634dc8d69e4c4": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "9aa34eaa1a784bcb8c5e92794c27ecc3": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "d7c2d402ee1e4e9190ffee2109a501f1": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_8f8518460bd1486f980634dc8d69e4c4", "placeholder": "\u200b", "style": "IPY_MODEL_9aa34eaa1a784bcb8c5e92794c27ecc3", "tabbable": null, "tooltip": null, "value": "tokenizer_config.json:\u2007100%"}}, "9d4d5924716049378a2ca41940476803": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "02b561a4fcbc4ff78b7b6b8ad0e896fb": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "4dd95b217ba34fae83ec141d5f2d0a4b": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_9d4d5924716049378a2ca41940476803", "placeholder": "\u200b", "style": "IPY_MODEL_02b561a4fcbc4ff78b7b6b8ad0e896fb", "tabbable": null, "tooltip": null, "value": "\u200726.0/26.0\u2007[00:00&lt;00:00,\u20074.58kB/s]"}}, "1d57c4e90fa74346bc732037c622ee7a": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "0688c83589ce4772b743149fd2bfabce": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_d7c2d402ee1e4e9190ffee2109a501f1", "IPY_MODEL_1588536871044d748718349a6c4ee375", "IPY_MODEL_4dd95b217ba34fae83ec141d5f2d0a4b"], "layout": "IPY_MODEL_1d57c4e90fa74346bc732037c622ee7a", "tabbable": null, "tooltip": null}}, "b9d76ce76d0f48c1bab169245339cc52": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "9f38f17b1a8243658f662b7c19cfc291": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "bd0f674f4df64ddaacd86248f49980ac": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_b9d76ce76d0f48c1bab169245339cc52", "max": 1042301.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_9f38f17b1a8243658f662b7c19cfc291", "tabbable": null, "tooltip": null, "value": 1042301.0}}, "8ece1ded3c794772bd30777f08400130": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "b95de56d838e48b3b5a86f0d727fc1ec": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "47b8804c26bc4c38bab0a6a682a080a0": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_8ece1ded3c794772bd30777f08400130", "placeholder": "\u200b", "style": "IPY_MODEL_b95de56d838e48b3b5a86f0d727fc1ec", "tabbable": null, "tooltip": null, "value": "vocab.json:\u2007100%"}}, "7fd52d89c83a450b94e80fba1f14b8bd": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "34a93cfc48a2454082ab382a7160b571": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "6fcd6a740f5043168d7697b1e10de7fe": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_7fd52d89c83a450b94e80fba1f14b8bd", "placeholder": "\u200b", "style": "IPY_MODEL_34a93cfc48a2454082ab382a7160b571", "tabbable": null, "tooltip": null, "value": "\u20071.04M/1.04M\u2007[00:00&lt;00:00,\u200734.6MB/s]"}}, "cb4cd3672cc54abc92f62efed4a71ded": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "27f33facb9e34151b8dce0f220cad270": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_47b8804c26bc4c38bab0a6a682a080a0", "IPY_MODEL_bd0f674f4df64ddaacd86248f49980ac", "IPY_MODEL_6fcd6a740f5043168d7697b1e10de7fe"], "layout": "IPY_MODEL_cb4cd3672cc54abc92f62efed4a71ded", "tabbable": null, "tooltip": null}}, "d9f88fcbeac145f9a24e0d101d91caf1": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "caa9978c2d3a4f2895f28c23db81350a": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "a5f89503b135454e9897be952ba25bf4": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_d9f88fcbeac145f9a24e0d101d91caf1", "max": 456318.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_caa9978c2d3a4f2895f28c23db81350a", "tabbable": null, "tooltip": null, "value": 456318.0}}, "a127007784d246c38bd3befe316fbaab": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "237bab3bdf234e028ba70f945b671f91": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "e77e435d623b4305bc459e7920424fd4": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_a127007784d246c38bd3befe316fbaab", "placeholder": "\u200b", "style": "IPY_MODEL_237bab3bdf234e028ba70f945b671f91", "tabbable": null, "tooltip": null, "value": "merges.txt:\u2007100%"}}, "82559a7cad044347acaeb563d9af1f85": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "a3a319cc10874775b1c0b734866a42ea": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "88889e572ad148ff87a8fffdb35a3afd": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_82559a7cad044347acaeb563d9af1f85", "placeholder": "\u200b", "style": "IPY_MODEL_a3a319cc10874775b1c0b734866a42ea", "tabbable": null, "tooltip": null, "value": "\u2007456k/456k\u2007[00:00&lt;00:00,\u200746.3MB/s]"}}, "b907ab58b5e24cba8615605ca344dba1": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "f9a0ca4b543f47e0987f20fc87d7135a": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_e77e435d623b4305bc459e7920424fd4", "IPY_MODEL_a5f89503b135454e9897be952ba25bf4", "IPY_MODEL_88889e572ad148ff87a8fffdb35a3afd"], "layout": "IPY_MODEL_b907ab58b5e24cba8615605ca344dba1", "tabbable": null, "tooltip": null}}, "ca0e337ba4e6481ba5728bae7f11f7d7": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "33010587d7c44fe3a1efdc8290ade333": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "6ffb62c075b844e3a584da1ba3604cd7": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_ca0e337ba4e6481ba5728bae7f11f7d7", "max": 1355256.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_33010587d7c44fe3a1efdc8290ade333", "tabbable": null, "tooltip": null, "value": 1355256.0}}, "4477b92240f94ad78c7423f91ac9eff5": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "d9d197ca62994c82947369a275890068": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "18ad3bc6dccd4d7ba1dcb8f5c67e96ea": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_4477b92240f94ad78c7423f91ac9eff5", "placeholder": "\u200b", "style": "IPY_MODEL_d9d197ca62994c82947369a275890068", "tabbable": null, "tooltip": null, "value": "tokenizer.json:\u2007100%"}}, "48458fc57304401b90253697e647dbd8": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "4315cacb349a45078d9a20a06d48a8a0": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "410b5e8ea7a74e53a4710f63bec5c46a": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_48458fc57304401b90253697e647dbd8", "placeholder": "\u200b", "style": "IPY_MODEL_4315cacb349a45078d9a20a06d48a8a0", "tabbable": null, "tooltip": null, "value": "\u20071.36M/1.36M\u2007[00:00&lt;00:00,\u200738.7MB/s]"}}, "3370afffa6ec450aa774360d4594d0f0": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "e4a807366560448bab5fc25c6deea451": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_18ad3bc6dccd4d7ba1dcb8f5c67e96ea", "IPY_MODEL_6ffb62c075b844e3a584da1ba3604cd7", "IPY_MODEL_410b5e8ea7a74e53a4710f63bec5c46a"], "layout": "IPY_MODEL_3370afffa6ec450aa774360d4594d0f0", "tabbable": null, "tooltip": null}}, "0540d008d7ad474995cbc1f97e500d77": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "18d9cd9c263d4202b75fbc5ead7a91da": {"model_name": "ProgressStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "bar_color": null, "description_width": ""}}, "95b475a20b204d5083a48b82a6f8382d": {"model_name": "FloatProgressModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_0540d008d7ad474995cbc1f97e500d77", "max": 50.0, "min": 0.0, "orientation": "horizontal", "style": "IPY_MODEL_18d9cd9c263d4202b75fbc5ead7a91da", "tabbable": null, "tooltip": null, "value": 50.0}}, "bcc763b0fc9147ed9c8db734601bd31a": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "6155a3097f1640089639d59a5fd01d55": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "6cee7db00ffb4c01bb6dd17bf0c55dc8": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_bcc763b0fc9147ed9c8db734601bd31a", "placeholder": "\u200b", "style": "IPY_MODEL_6155a3097f1640089639d59a5fd01d55", "tabbable": null, "tooltip": null, "value": "100%"}}, "d95212bf72bc45f9b942f1f07c7b3497": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "17da7e9af1c347c28c312e42a4efece7": {"model_name": "HTMLStyleModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", "background": null, "description_width": "", "font_size": null, "text_color": null}}, "805a8f8fcf3e4378a39ce356b3c7c23e": {"model_name": "HTMLModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HTMLView", "description": "", "description_allow_html": false, "layout": "IPY_MODEL_d95212bf72bc45f9b942f1f07c7b3497", "placeholder": "\u200b", "style": "IPY_MODEL_17da7e9af1c347c28c312e42a4efece7", "tabbable": null, "tooltip": null, "value": "\u200750/50\u2007[00:02&lt;00:00,\u200721.71it/s]"}}, "311dee65e0594c0c9be87cac3ff30b6c": {"model_name": "LayoutModel", "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "state": {"_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border_bottom": null, "border_left": null, "border_right": null, "border_top": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null}}, "008a6d643cb74e9da094f214bdade5b2": {"model_name": "HBoxModel", "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "state": {"_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", "_view_name": "HBoxView", "box_style": "", "children": ["IPY_MODEL_6cee7db00ffb4c01bb6dd17bf0c55dc8", "IPY_MODEL_95b475a20b204d5083a48b82a6f8382d", "IPY_MODEL_805a8f8fcf3e4378a39ce356b3c7c23e"], "layout": "IPY_MODEL_311dee65e0594c0c9be87cac3ff30b6c", "tabbable": null, "tooltip": null}}}, "version_major": 2, "version_minor": 0}
 </script></section>
 </section>
 </section>
diff --git a/generated/demos/Main_Demo.ipynb b/generated/demos/Main_Demo.ipynb
index db3be41d2..5a8ead9da 100644
--- a/generated/demos/Main_Demo.ipynb
+++ b/generated/demos/Main_Demo.ipynb
@@ -48,10 +48,10 @@
    "execution_count": 1,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:41:45.440694Z",
-     "iopub.status.busy": "2024-09-05T17:41:45.440516Z",
-     "iopub.status.idle": "2024-09-05T17:41:45.450621Z",
-     "shell.execute_reply": "2024-09-05T17:41:45.450126Z"
+     "iopub.execute_input": "2024-09-10T17:15:24.499639Z",
+     "iopub.status.busy": "2024-09-10T17:15:24.499030Z",
+     "iopub.status.idle": "2024-09-10T17:15:24.509656Z",
+     "shell.execute_reply": "2024-09-10T17:15:24.509238Z"
     }
    },
    "outputs": [],
@@ -89,10 +89,10 @@
    "execution_count": 2,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:41:45.452777Z",
-     "iopub.status.busy": "2024-09-05T17:41:45.452429Z",
-     "iopub.status.idle": "2024-09-05T17:41:45.571224Z",
-     "shell.execute_reply": "2024-09-05T17:41:45.570632Z"
+     "iopub.execute_input": "2024-09-10T17:15:24.511498Z",
+     "iopub.status.busy": "2024-09-10T17:15:24.511335Z",
+     "iopub.status.idle": "2024-09-10T17:15:24.631630Z",
+     "shell.execute_reply": "2024-09-10T17:15:24.631062Z"
     }
    },
    "outputs": [
@@ -119,28 +119,28 @@
    "execution_count": 3,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:41:45.573454Z",
-     "iopub.status.busy": "2024-09-05T17:41:45.573082Z",
-     "iopub.status.idle": "2024-09-05T17:41:47.022059Z",
-     "shell.execute_reply": "2024-09-05T17:41:47.021391Z"
+     "iopub.execute_input": "2024-09-10T17:15:24.633900Z",
+     "iopub.status.busy": "2024-09-10T17:15:24.633545Z",
+     "iopub.status.idle": "2024-09-10T17:15:25.990540Z",
+     "shell.execute_reply": "2024-09-10T17:15:25.990012Z"
     }
    },
    "outputs": [
     {
      "data": {
       "text/html": [
-       "<div id=\"circuits-vis-2e3c0ab6-b350\" style=\"margin: 15px 0;\"/>\n",
+       "<div id=\"circuits-vis-8b570228-4d3b\" style=\"margin: 15px 0;\"/>\n",
        "    <script crossorigin type=\"module\">\n",
        "    import { render, Hello } from \"https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js\";\n",
        "    render(\n",
-       "      \"circuits-vis-2e3c0ab6-b350\",\n",
+       "      \"circuits-vis-8b570228-4d3b\",\n",
        "      Hello,\n",
        "      {\"name\": \"Neel\"}\n",
        "    )\n",
        "    </script>"
       ],
       "text/plain": [
-       "<circuitsvis.utils.render.RenderedHTML at 0x7f209d2f4950>"
+       "<circuitsvis.utils.render.RenderedHTML at 0x7f283d06eb10>"
       ]
      },
      "execution_count": 3,
@@ -159,10 +159,10 @@
    "execution_count": 4,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:41:47.024419Z",
-     "iopub.status.busy": "2024-09-05T17:41:47.024148Z",
-     "iopub.status.idle": "2024-09-05T17:41:47.328364Z",
-     "shell.execute_reply": "2024-09-05T17:41:47.327856Z"
+     "iopub.execute_input": "2024-09-10T17:15:25.992696Z",
+     "iopub.status.busy": "2024-09-10T17:15:25.992455Z",
+     "iopub.status.idle": "2024-09-10T17:15:26.286880Z",
+     "shell.execute_reply": "2024-09-10T17:15:26.286319Z"
     }
    },
    "outputs": [],
@@ -184,10 +184,10 @@
    "execution_count": 5,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:41:47.330913Z",
-     "iopub.status.busy": "2024-09-05T17:41:47.330650Z",
-     "iopub.status.idle": "2024-09-05T17:41:48.854787Z",
-     "shell.execute_reply": "2024-09-05T17:41:48.854217Z"
+     "iopub.execute_input": "2024-09-10T17:15:26.289509Z",
+     "iopub.status.busy": "2024-09-10T17:15:26.289074Z",
+     "iopub.status.idle": "2024-09-10T17:15:27.816835Z",
+     "shell.execute_reply": "2024-09-10T17:15:27.816293Z"
     }
    },
    "outputs": [],
@@ -212,17 +212,17 @@
    "execution_count": 6,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:41:48.857143Z",
-     "iopub.status.busy": "2024-09-05T17:41:48.856952Z",
-     "iopub.status.idle": "2024-09-05T17:41:48.860659Z",
-     "shell.execute_reply": "2024-09-05T17:41:48.860123Z"
+     "iopub.execute_input": "2024-09-10T17:15:27.819301Z",
+     "iopub.status.busy": "2024-09-10T17:15:27.818946Z",
+     "iopub.status.idle": "2024-09-10T17:15:27.822835Z",
+     "shell.execute_reply": "2024-09-10T17:15:27.822395Z"
     }
    },
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "<torch.autograd.grad_mode.set_grad_enabled at 0x7f208dc6d3d0>"
+       "<torch.autograd.grad_mode.set_grad_enabled at 0x7f282d327510>"
       ]
      },
      "execution_count": 6,
@@ -246,10 +246,10 @@
    "execution_count": 7,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:41:48.862711Z",
-     "iopub.status.busy": "2024-09-05T17:41:48.862367Z",
-     "iopub.status.idle": "2024-09-05T17:41:48.867622Z",
-     "shell.execute_reply": "2024-09-05T17:41:48.867089Z"
+     "iopub.execute_input": "2024-09-10T17:15:27.824836Z",
+     "iopub.status.busy": "2024-09-10T17:15:27.824548Z",
+     "iopub.status.idle": "2024-09-10T17:15:27.829709Z",
+     "shell.execute_reply": "2024-09-10T17:15:27.829259Z"
     }
    },
    "outputs": [],
@@ -301,10 +301,10 @@
    "execution_count": 8,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:41:48.869668Z",
-     "iopub.status.busy": "2024-09-05T17:41:48.869326Z",
-     "iopub.status.idle": "2024-09-05T17:41:48.872222Z",
-     "shell.execute_reply": "2024-09-05T17:41:48.871756Z"
+     "iopub.execute_input": "2024-09-10T17:15:27.831830Z",
+     "iopub.status.busy": "2024-09-10T17:15:27.831447Z",
+     "iopub.status.idle": "2024-09-10T17:15:27.834444Z",
+     "shell.execute_reply": "2024-09-10T17:15:27.833878Z"
     }
    },
    "outputs": [],
@@ -317,10 +317,10 @@
    "execution_count": 9,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:41:48.874276Z",
-     "iopub.status.busy": "2024-09-05T17:41:48.873934Z",
-     "iopub.status.idle": "2024-09-05T17:41:50.053211Z",
-     "shell.execute_reply": "2024-09-05T17:41:50.052584Z"
+     "iopub.execute_input": "2024-09-10T17:15:27.836518Z",
+     "iopub.status.busy": "2024-09-10T17:15:27.836188Z",
+     "iopub.status.idle": "2024-09-10T17:15:28.701672Z",
+     "shell.execute_reply": "2024-09-10T17:15:28.700959Z"
     }
    },
    "outputs": [
@@ -354,10 +354,10 @@
    "execution_count": 10,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:41:50.055420Z",
-     "iopub.status.busy": "2024-09-05T17:41:50.055192Z",
-     "iopub.status.idle": "2024-09-05T17:41:50.375631Z",
-     "shell.execute_reply": "2024-09-05T17:41:50.375068Z"
+     "iopub.execute_input": "2024-09-10T17:15:28.703925Z",
+     "iopub.status.busy": "2024-09-10T17:15:28.703556Z",
+     "iopub.status.idle": "2024-09-10T17:15:29.020416Z",
+     "shell.execute_reply": "2024-09-10T17:15:29.019785Z"
     }
    },
    "outputs": [
@@ -399,10 +399,10 @@
    "execution_count": 11,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:41:50.378189Z",
-     "iopub.status.busy": "2024-09-05T17:41:50.377798Z",
-     "iopub.status.idle": "2024-09-05T17:41:50.654498Z",
-     "shell.execute_reply": "2024-09-05T17:41:50.653899Z"
+     "iopub.execute_input": "2024-09-10T17:15:29.022678Z",
+     "iopub.status.busy": "2024-09-10T17:15:29.022498Z",
+     "iopub.status.idle": "2024-09-10T17:15:29.303069Z",
+     "shell.execute_reply": "2024-09-10T17:15:29.302413Z"
     }
    },
    "outputs": [
@@ -440,10 +440,10 @@
    "execution_count": 12,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:41:50.657195Z",
-     "iopub.status.busy": "2024-09-05T17:41:50.656791Z",
-     "iopub.status.idle": "2024-09-05T17:41:50.661566Z",
-     "shell.execute_reply": "2024-09-05T17:41:50.661095Z"
+     "iopub.execute_input": "2024-09-10T17:15:29.305735Z",
+     "iopub.status.busy": "2024-09-10T17:15:29.305395Z",
+     "iopub.status.idle": "2024-09-10T17:15:29.309808Z",
+     "shell.execute_reply": "2024-09-10T17:15:29.309338Z"
     }
    },
    "outputs": [
@@ -468,10 +468,10 @@
    "execution_count": 13,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:41:50.663753Z",
-     "iopub.status.busy": "2024-09-05T17:41:50.663405Z",
-     "iopub.status.idle": "2024-09-05T17:41:50.910499Z",
-     "shell.execute_reply": "2024-09-05T17:41:50.909847Z"
+     "iopub.execute_input": "2024-09-10T17:15:29.311691Z",
+     "iopub.status.busy": "2024-09-10T17:15:29.311401Z",
+     "iopub.status.idle": "2024-09-10T17:15:29.385406Z",
+     "shell.execute_reply": "2024-09-10T17:15:29.384822Z"
     }
    },
    "outputs": [
@@ -485,18 +485,18 @@
     {
      "data": {
       "text/html": [
-       "<div id=\"circuits-vis-a7b9010c-0065\" style=\"margin: 15px 0;\"/>\n",
+       "<div id=\"circuits-vis-7b1e6e9f-a1f2\" style=\"margin: 15px 0;\"/>\n",
        "    <script crossorigin type=\"module\">\n",
        "    import { render, AttentionPatterns } from \"https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js\";\n",
        "    render(\n",
-       "      \"circuits-vis-a7b9010c-0065\",\n",
+       "      \"circuits-vis-7b1e6e9f-a1f2\",\n",
        "      AttentionPatterns,\n",
        "      {\"tokens\": [\"<|endoftext|>\", \"Natural\", \" language\", \" processing\", \" tasks\", \",\", \" such\", \" as\", \" question\", \" answering\", \",\", \" machine\", \" translation\", \",\", \" reading\", \" comprehension\", \",\", \" and\", \" summar\", \"ization\", \",\", \" are\", \" typically\", \" approached\", \" with\", \" supervised\", \" learning\", \" on\", \" tasks\", \"pe\", \"cific\", \" datasets\", \".\"], \"attention\": [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9639418125152588, 0.03605816140770912, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8389372825622559, 0.11828788369894028, 0.04277484491467476, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.47436124086380005, 0.13382026553153992, 0.27371731400489807, 0.11810113489627838, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.35606440901756287, 0.10184910148382187, 0.23054221272468567, 0.20397402346134186, 0.10757026076316833, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6660143733024597, 0.16866375505924225, 0.04535674676299095, 0.038855016231536865, 0.0677548199892044, 0.013355279341340065, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.38626962900161743, 0.285109281539917, 0.07609007507562637, 0.05908379331231117, 0.07223352044820786, 0.03979635238647461, 0.08141742646694183, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3775395154953003, 0.1883881539106369, 0.11723985522985458, 0.08685600012540817, 0.0666918158531189, 0.03500017523765564, 0.09693004935979843, 0.03135441616177559, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4869752824306488, 0.06781317293643951, 0.07952874153852463, 0.0848078578710556, 0.1590261608362198, 0.029577823355793953, 0.025685923174023628, 0.016474608331918716, 0.05011039599776268, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2906550168991089, 0.0401349738240242, 0.14614859223365784, 0.09940590709447861, 0.1538919061422348, 0.039001598954200745, 0.024988971650600433, 0.03184127062559128, 0.10222824662923813, 0.0717034563422203, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.39624103903770447, 0.09694176912307739, 0.027270659804344177, 0.02355135791003704, 0.03723449259996414, 0.006502415519207716, 0.08118756860494614, 0.013088458217680454, 0.06990590691566467, 0.24043099582195282, 0.007645336911082268, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.24864791333675385, 0.1380205899477005, 0.0923532024025917, 0.08676131069660187, 0.1381969302892685, 0.05914197862148285, 0.03223859518766403, 0.03158237785100937, 0.030489422380924225, 0.03873484209179878, 0.06671839207410812, 0.037114467471838, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1914844512939453, 0.1617259681224823, 0.07445938140153885, 0.07740949839353561, 0.021961113438010216, 0.03392130509018898, 0.05125021934509277, 0.01951923966407776, 0.03132446110248566, 0.04020152986049652, 0.038742680102586746, 0.21578852832317352, 0.042211681604385376, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3704318404197693, 0.08681417256593704, 0.02458467148244381, 0.021616315469145775, 0.032388731837272644, 0.005422737915068865, 0.0727522075176239, 0.0112727927044034, 0.06329693645238876, 0.21726809442043304, 0.006367155350744724, 0.029603807255625725, 0.05099846422672272, 0.007182050962001085, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1973765343427658, 0.04603995382785797, 0.0439998134970665, 0.1337345838546753, 0.05424821376800537, 0.025475721806287766, 0.027563506737351418, 0.021570932120084763, 0.051718227565288544, 0.06458097696304321, 0.02806464210152626, 0.23551592230796814, 0.019129814580082893, 0.029963519424200058, 0.021017681807279587, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08907236158847809, 0.01928834617137909, 0.1665353924036026, 0.07281264662742615, 0.04738640412688255, 0.024487905204296112, 0.028987322002649307, 0.019370367750525475, 0.026673022657632828, 0.07316635549068451, 0.025704585015773773, 0.04242359474301338, 0.058694612234830856, 0.028932694345712662, 0.18119072914123535, 0.09527372568845749, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2816729247570038, 0.06441289931535721, 0.018008552491664886, 0.01616962067782879, 0.023183872923254967, 0.0037532988935709, 0.05472247675061226, 0.007909760810434818, 0.046164702624082565, 0.16947267949581146, 0.004361647181212902, 0.021011337637901306, 0.0354907400906086, 0.004932564217597246, 0.0955522358417511, 0.14726325869560242, 0.005917454604059458, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.21305488049983978, 0.0591236874461174, 0.03382088243961334, 0.02747686579823494, 0.02839355543255806, 0.008422904647886753, 0.0400853231549263, 0.011629256419837475, 0.05295189097523689, 0.15404635667800903, 0.009831804782152176, 0.03610190004110336, 0.04737289249897003, 0.011069180443882942, 0.09972473233938217, 0.1397135704755783, 0.013185348361730576, 0.013994931243360043, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15871697664260864, 0.04387888312339783, 0.08712151646614075, 0.08998466283082962, 0.030738582834601402, 0.0341489352285862, 0.024917256087064743, 0.03139195218682289, 0.024823859333992004, 0.019790329039096832, 0.03625484183430672, 0.020694412291049957, 0.042840685695409775, 0.03820899501442909, 0.06234658882021904, 0.10919700562953949, 0.0413760244846344, 0.04916759952902794, 0.054400913417339325, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10485509783029556, 0.12122292071580887, 0.0648748055100441, 0.08768714964389801, 0.03434053063392639, 0.01748395338654518, 0.034151818603277206, 0.015289156697690487, 0.023312119767069817, 0.028306512162089348, 0.01872047781944275, 0.028111934661865234, 0.041905295103788376, 0.0209895521402359, 0.04678506404161453, 0.08659638464450836, 0.023631852120161057, 0.024273162707686424, 0.16702400147914886, 0.010438223369419575, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.22901973128318787, 0.05184381827712059, 0.013585173524916172, 0.012337238527834415, 0.018005413934588432, 0.0027703619562089443, 0.042381297796964645, 0.005856257397681475, 0.03614485636353493, 0.13039222359657288, 0.0031534270383417606, 0.01567256823182106, 0.027800394222140312, 0.003554322523996234, 0.07460816204547882, 0.11298283189535141, 0.004272268619388342, 0.006832208018749952, 0.18569746613502502, 0.018073637038469315, 0.005016352981328964, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.18869927525520325, 0.03438714146614075, 0.022344734519720078, 0.019972724840044975, 0.016354026272892952, 0.0068565551191568375, 0.020859038457274437, 0.005695997271686792, 0.03415916487574577, 0.07260986417531967, 0.007857211865484715, 0.018040239810943604, 0.026904456317424774, 0.009020394645631313, 0.06876447051763535, 0.17578734457492828, 0.010720067657530308, 0.00928453542292118, 0.1925639659166336, 0.025180332362651825, 0.012639074586331844, 0.021299341693520546, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1195831298828125, 0.022259412333369255, 0.032947149127721786, 0.020170224830508232, 0.035653311759233475, 0.013459913432598114, 0.017516475170850754, 0.010057873092591763, 0.025856440886855125, 0.05955953523516655, 0.015084508806467056, 0.015008730813860893, 0.053174685686826706, 0.016597608104348183, 0.041555255651474, 0.131293386220932, 0.01929667219519615, 0.015855036675930023, 0.1792508363723755, 0.01618383638560772, 0.02229553461074829, 0.015463392250239849, 0.10187702625989914, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14615531265735626, 0.026727521792054176, 0.01662455126643181, 0.018987685441970825, 0.06278638541698456, 0.015317163430154324, 0.019792240113019943, 0.01422776561230421, 0.025458168238401413, 0.045303549617528915, 0.016364356502890587, 0.037493038922548294, 0.0132886478677392, 0.017496541142463684, 0.0399458184838295, 0.05881758779287338, 0.01926097832620144, 0.024616025388240814, 0.038219697773456573, 0.02157779224216938, 0.02094990760087967, 0.07973217219114304, 0.05017608404159546, 0.17068105936050415, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.11321667581796646, 0.03674636408686638, 0.011786575429141521, 0.010274871252477169, 0.020370664075016975, 0.00524388812482357, 0.015918847173452377, 0.005266785155981779, 0.02489173412322998, 0.06593258678913116, 0.005933663807809353, 0.018209032714366913, 0.021020200103521347, 0.0066674984991550446, 0.034828800708055496, 0.13742128014564514, 0.007927048951387405, 0.008618664927780628, 0.1137719452381134, 0.013557437807321548, 0.009277833625674248, 0.026121344417333603, 0.08499342203140259, 0.19073909521102905, 0.011263742111623287, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.13337711989879608, 0.026216603815555573, 0.03827154263854027, 0.07152578234672546, 0.05317767709493637, 0.013925841078162193, 0.007084188051521778, 0.013450146652758121, 0.009841442108154297, 0.011789779178798199, 0.013537583872675896, 0.038154955953359604, 0.041933052241802216, 0.013882285915315151, 0.03707146272063255, 0.138384610414505, 0.014846334233880043, 0.03156952187418938, 0.05598173290491104, 0.015536684542894363, 0.01595635898411274, 0.045455560088157654, 0.01669965498149395, 0.02532576024532318, 0.03671892359852791, 0.08028541505336761, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10608974099159241, 0.019107727333903313, 0.024468660354614258, 0.027496395632624626, 0.016365809366106987, 0.0050114234909415245, 0.010413105599582195, 0.006081144325435162, 0.005301064345985651, 0.011143166571855545, 0.004565386101603508, 0.01896991953253746, 0.004321120213717222, 0.00481497822329402, 0.02940940298140049, 0.028682034462690353, 0.005097254645079374, 0.0072343479841947556, 0.03412593528628349, 0.010370595380663872, 0.005643266253173351, 0.007283586077392101, 0.029389560222625732, 0.010038800537586212, 0.009134513325989246, 0.546663224697113, 0.012777911499142647, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10467305034399033, 0.03321940079331398, 0.015341250225901604, 0.009373549371957779, 0.026595456525683403, 0.0057878089137375355, 0.013571349903941154, 0.004554884973913431, 0.028058893978595734, 0.026107225567102432, 0.006353434175252914, 0.013315831311047077, 0.026628265157341957, 0.006888872943818569, 0.06204749271273613, 0.05890703946352005, 0.008068050257861614, 0.007557098753750324, 0.0852278545498848, 0.017075754702091217, 0.009256886318325996, 0.019695758819580078, 0.12617813050746918, 0.13061513006687164, 0.011351036839187145, 0.08984372764825821, 0.04638150706887245, 0.007325289770960808, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08561903238296509, 0.021438946947455406, 0.056412599980831146, 0.0570666640996933, 0.019802208989858627, 0.00672749662771821, 0.005809164140373468, 0.004516261164098978, 0.0031647481955587864, 0.01761520653963089, 0.006174598820507526, 0.08767975866794586, 0.012299864552915096, 0.006350455805659294, 0.017522143200039864, 0.14295215904712677, 0.006585489492863417, 0.007875688374042511, 0.030078941956162453, 0.01390750240534544, 0.0073766945861279964, 0.007684784941375256, 0.02216084487736225, 0.012385032139718533, 0.011890063062310219, 0.08669772744178772, 0.19902527332305908, 0.01359447930008173, 0.029586246237158775, 0.0, 0.0, 0.0, 0.0], [0.14064587652683258, 0.013298707082867622, 0.015702949836850166, 0.017357859760522842, 0.02233150415122509, 0.029672270640730858, 0.04172082245349884, 0.01899542473256588, 0.03827710077166557, 0.048635613173246384, 0.03094690665602684, 0.016023898497223854, 0.020880894735455513, 0.032438233494758606, 0.030558118596673012, 0.022808339446783066, 0.03537759929895401, 0.031451594084501266, 0.03497113287448883, 0.01867910660803318, 0.03821907937526703, 0.02257886528968811, 0.06819561868906021, 0.04214095324277878, 0.028620855882763863, 0.03775003179907799, 0.018578052520751953, 0.03376871719956398, 0.03641696274280548, 0.012956855818629265, 0.0, 0.0, 0.0], [0.07168620079755783, 0.06924446672201157, 0.019306905567646027, 0.014161979779601097, 0.016823193058371544, 0.0193806029856205, 0.019257402047514915, 0.0220036581158638, 0.013706518337130547, 0.03578377887606621, 0.018465185537934303, 0.05207168683409691, 0.020085178315639496, 0.01986212469637394, 0.02066212333738804, 0.04725164547562599, 0.021076705306768417, 0.036787249147892, 0.024324089288711548, 0.0038275483530014753, 0.023920675739645958, 0.008533227257430553, 0.026241622865200043, 0.027380064129829407, 0.03461199253797531, 0.022884156554937363, 0.10047907382249832, 0.06913496553897858, 0.025474581867456436, 0.06495600938796997, 0.03061538189649582, 0.0, 0.0], [0.0691077709197998, 0.0370122492313385, 0.03862115740776062, 0.05933326855301857, 0.015923550352454185, 0.007918563671410084, 0.010371048003435135, 0.0066156634129583836, 0.002520075300708413, 0.026019366458058357, 0.007905230857431889, 0.02965201810002327, 0.04000623896718025, 0.008451344445347786, 0.010741154663264751, 0.05027563124895096, 0.009428859688341618, 0.013601033948361874, 0.05036921799182892, 0.03176714852452278, 0.010793027468025684, 0.007216801401227713, 0.006478427443653345, 0.01480061188340187, 0.021585967391729355, 0.15769484639167786, 0.08884759992361069, 0.0190169308334589, 0.02972934953868389, 0.03316137194633484, 0.050883661955595016, 0.034150850027799606, 0.0], [0.14375509321689606, 0.01681104488670826, 0.009386662393808365, 0.006830313708633184, 0.011656845919787884, 0.0015672279987484217, 0.019711481407284737, 0.002398042706772685, 0.021235886961221695, 0.04683680087327957, 0.0016905148513615131, 0.0058271917514503, 0.011979999952018261, 0.0018251396249979734, 0.042313333600759506, 0.054913729429244995, 0.0021786566358059645, 0.0024170097894966602, 0.09604065865278244, 0.005752880126237869, 0.0025577889755368233, 0.0071212234906852245, 0.08889931440353394, 0.10852081328630447, 0.005179054103791714, 0.03657734394073486, 0.02471994049847126, 0.0037347625475376844, 0.031077096238732338, 0.016887947916984558, 0.09450862556695938, 0.07171519845724106, 0.0033723204396665096]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0004246663593221456, 0.9995753169059753, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0005621908348985016, 0.016407281160354614, 0.9830306172370911, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0011627554195001721, 0.021681973710656166, 0.0037620372604578733, 0.9733933210372925, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [3.7244368286337703e-05, 0.00017202268645633012, 0.0002814392792060971, 0.0027421435806900263, 0.9967671632766724, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00826845783740282, 0.00023985625011846423, 7.361917960224673e-05, 6.43773382762447e-05, 0.0001756635756464675, 0.9911779761314392, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0012215041788294911, 0.005400444380939007, 0.0016716312384232879, 0.00040775578236207366, 0.0006163655198179185, 0.001093117636628449, 0.989589273929596, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0012459794525057077, 0.0009121228358708322, 0.0005976726533845067, 0.00013656896771863103, 0.00033041107235476375, 0.001572280889376998, 0.0038808276876807213, 0.9913240671157837, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00028217516955919564, 0.004068182315677404, 0.0026605194434523582, 0.0013093105517327785, 0.008030476048588753, 0.00028790938085876405, 0.00022922940843272954, 0.0003948427038267255, 0.9827372431755066, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [3.47392306139227e-05, 0.0003953832492697984, 0.00013272685464471579, 0.0002585221081972122, 0.001085555530153215, 9.198026964440942e-05, 0.000326707202475518, 0.000542744412086904, 0.006105918437242508, 0.9910256266593933, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0033785062842071056, 5.0908744015032426e-05, 1.6452069758088328e-05, 1.69261602422921e-05, 4.18141353293322e-05, 0.49394020438194275, 0.00012981283362023532, 0.0008837337954901159, 3.221205042791553e-05, 2.7252061045146547e-05, 0.5014821887016296, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [8.416055788984522e-05, 0.0013417234877124429, 0.001261359779164195, 0.0021450764033943415, 0.004042366519570351, 0.00048305385280400515, 0.0001158266604761593, 0.00015203595103230327, 2.6925330530502833e-05, 0.00012675137259066105, 0.00031289938488043845, 0.9899077415466309, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0003782230196520686, 0.0009837561519816518, 0.03934125974774361, 0.002732245484367013, 0.0036680190823972225, 0.00011039189848816022, 0.00012931032688356936, 0.00021743457182310522, 0.00010623293928802013, 0.0007748182397335768, 6.647665577474982e-05, 0.0003148667747154832, 0.9511768817901611, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0021017189137637615, 2.443686025799252e-05, 7.78878529672511e-06, 8.651618372823577e-06, 2.014001256611664e-05, 0.2997134327888489, 7.525274122599512e-05, 0.0004898307379335165, 1.845947736001108e-05, 1.534453986096196e-05, 0.32833898067474365, 4.175798676442355e-05, 6.469185791502241e-06, 0.3691376745700836, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0001496832846896723, 0.00011296618322376162, 0.00036294886376708746, 0.00018591222760733217, 0.00016460890765301883, 4.1432296711718664e-05, 2.8764745366061106e-05, 7.786958303768188e-05, 0.0009200992644764483, 0.0103401904925704, 2.7572339604375884e-05, 1.7833237507147714e-05, 0.0003305452992208302, 2.4375704015255906e-05, 0.9872152805328369, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00010753834794741124, 0.0021781916730105877, 0.002042611828073859, 0.004251930397003889, 0.006989913992583752, 2.511881393729709e-05, 0.0007779046427458525, 0.0005783551605418324, 0.00293784704990685, 0.033225417137145996, 1.71992760442663e-05, 0.0008936497615650296, 0.0015238587511703372, 1.4656818166258745e-05, 0.00622264388948679, 0.9382131695747375, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.001363488263450563, 1.3226085684436839e-05, 4.013950274384115e-06, 4.803066076419782e-06, 1.0257452231599018e-05, 0.19665955007076263, 4.527267810772173e-05, 0.0002776262117549777, 1.1714444553945214e-05, 9.473311365582049e-06, 0.22919613122940063, 2.6430649086250924e-05, 4.101847935089609e-06, 0.26576095819473267, 8.515355148119852e-06, 4.536029791779583e-06, 0.30659976601600647, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0008924751309677958, 0.00013490811397787184, 4.779841037816368e-05, 5.8037036069436e-05, 0.00010480164928594604, 0.012799090705811977, 0.0007168247830122709, 0.032579511404037476, 2.6449959477758966e-05, 0.00011185064795427024, 0.011884261853992939, 4.010266638942994e-05, 5.555404641199857e-05, 0.012377863749861717, 0.00010783471225295216, 5.4043663112679496e-05, 0.013122126460075378, 0.914886474609375, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [5.075309672974981e-06, 4.396129952510819e-05, 3.398501576157287e-05, 7.940301293274388e-05, 5.477911327034235e-05, 7.921535711830074e-07, 9.313323971582577e-06, 7.727071533736307e-06, 8.597254054620862e-05, 0.00012274031178094447, 5.141479277881444e-07, 1.702793611002562e-06, 3.83417536795605e-05, 4.4509667418424215e-07, 0.00013928221596870571, 0.0003275797644164413, 3.9947968843989656e-07, 3.948126504838001e-06, 0.999043881893158, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [9.346180740976706e-05, 0.0018395738443359733, 0.0025233125779777765, 0.018087238073349, 0.0029363748617470264, 0.00027335810591466725, 4.872974750469439e-05, 0.0004212774510961026, 0.00015624347724951804, 0.0009748333832249045, 0.00020533586211968213, 0.0010228825267404318, 0.0019548034761101007, 0.0001947036653291434, 0.0011294205905869603, 0.0016656103543937206, 0.00018734042532742023, 0.0009503448964096606, 0.0004455175076145679, 0.964889645576477, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0010859702015295625, 8.512333806720562e-06, 2.5307288069598144e-06, 3.0625205909018405e-06, 5.975129170110449e-06, 0.13292615115642548, 3.345326331327669e-05, 0.00018891248328145593, 8.477756637148559e-06, 6.540415142808342e-06, 0.1644592434167862, 1.813003655115608e-05, 3.0627427349827485e-06, 0.19719170033931732, 6.4298355937353335e-06, 3.444739604674396e-06, 0.23317119479179382, 0.0022796259727329016, 3.713432988661225e-06, 3.52880269929301e-05, 0.2685585916042328, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0004197956877760589, 0.00011805014219135046, 0.00014240501332096756, 3.796788223553449e-05, 0.00019043161591980606, 0.00176512252073735, 0.00057098304387182, 0.0005008855368942022, 8.840763621265069e-05, 0.0001420867774868384, 0.001663984265178442, 3.348111204104498e-05, 2.4413982828264125e-05, 0.00175465049687773, 6.520311580970883e-05, 2.414266964478884e-05, 0.0018299149814993143, 0.0015691040316596627, 3.974881110480055e-05, 0.00015712414460722357, 0.0018554475391283631, 0.9870065450668335, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [7.501777145080268e-05, 0.0014379826607182622, 6.345151632558554e-05, 0.00010864829528145492, 0.00015633167640771717, 3.2101231681735953e-06, 0.002203279873356223, 0.00022076554887462407, 5.240297468844801e-05, 4.8815789341460913e-05, 2.2644592263532104e-06, 1.5327233995776623e-05, 4.157140665483894e-06, 2.022839225901407e-06, 6.296796982496744e-06, 4.848669050261378e-05, 1.9928991150663933e-06, 3.247004497097805e-05, 0.0012695090845227242, 1.9632627299870364e-05, 1.8090934190695407e-06, 0.0005810288130305707, 0.9936450719833374, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [7.450151315424591e-05, 0.0006139386096037924, 0.0009361191187053919, 0.0008487795130349696, 0.002850631484761834, 1.0365051821281668e-05, 0.00021614256547763944, 0.00017397530609741807, 0.0020508402958512306, 0.005805297289043665, 8.055229955061805e-06, 8.086592424660921e-05, 0.0007702436414547265, 7.28818440620671e-06, 0.0010576159693300724, 0.002275596372783184, 6.663255135208601e-06, 0.00011621018347796053, 0.0005972451181150973, 8.736297604627907e-05, 6.332331849989714e-06, 6.096452852943912e-05, 6.090577517170459e-05, 0.9812840819358826, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0005848742439411581, 0.00015909200010355562, 1.083647021005163e-05, 7.365475903498009e-05, 0.00011349534906912595, 0.0008256662986241281, 0.00031911072437651455, 0.018529996275901794, 1.0226598533336073e-05, 4.9587219109525904e-05, 0.0007716424879617989, 4.454819281818345e-05, 9.865034371614456e-06, 0.0008067170856520534, 2.2673882995150052e-05, 1.2464139217627235e-05, 0.0008449103333987296, 0.008790111169219017, 3.5792407288681716e-05, 3.66286258213222e-05, 0.0008917527738958597, 0.0010791773675009608, 0.0003708462754730135, 0.0001083713723346591, 0.9654979109764099, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [1.6459925973322242e-05, 0.00017152438522316515, 3.2083211408462375e-05, 0.00010234172805212438, 0.002631882205605507, 9.886184670904186e-06, 3.250848749303259e-05, 3.7417539715534076e-05, 0.00012631539721041918, 4.9912112444872037e-05, 8.302540663862601e-06, 8.443430851912126e-05, 3.127968739136122e-05, 7.633363566128537e-06, 1.0101352927449625e-05, 5.6673809012863785e-05, 7.442129117407603e-06, 2.7689655325957574e-05, 1.841835728555452e-05, 2.879437261071871e-06, 6.840427886345424e-06, 4.279875156498747e-06, 0.00043176551116630435, 0.0001761750172590837, 8.995599637273699e-05, 0.9958257675170898, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [2.6368099497631192e-05, 0.00018008743063546717, 0.0001808296947274357, 0.00030464722658507526, 0.00039390946039929986, 4.674849333241582e-05, 2.7231719286646694e-05, 4.8734953452367336e-05, 0.00029131697374396026, 0.0004206168814562261, 3.804643711191602e-05, 0.0002524516312405467, 5.6067383411573246e-05, 3.82037615054287e-05, 0.0015365129802376032, 0.001253720954991877, 3.593428846215829e-05, 2.303666406078264e-05, 0.0001803626073524356, 0.0001226610183948651, 3.517790537443943e-05, 6.924685294507071e-05, 0.00011267305671935901, 0.0008507389575242996, 0.0001436190214008093, 0.00023528003657702357, 0.9930958151817322, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0007048255065456033, 5.675383727066219e-05, 3.351289251440903e-06, 7.472657671314664e-06, 1.785946005838923e-05, 0.0008156524272635579, 4.9529528041603044e-05, 0.0013530971482396126, 3.741759792319499e-05, 0.00014378006744664162, 0.0007639332907274365, 9.991676051868126e-06, 2.8844410735473502e-06, 0.0007552222232334316, 0.00010159592784475535, 3.1205443065118743e-06, 0.000806094438303262, 0.0011406756239011884, 1.4343090697366279e-05, 9.910167136695236e-06, 0.0008649186929687858, 8.663265907671303e-05, 3.4695403883233666e-05, 0.00010265821038046852, 0.007663471158593893, 4.187891317997128e-05, 3.4908698580693454e-06, 0.9844048619270325, 0.0, 0.0, 0.0, 0.0, 0.0], [2.0288332962081768e-05, 2.954272713395767e-05, 5.0375034334138036e-05, 0.00097783247474581, 0.3728252649307251, 6.6613192757358775e-06, 1.573364352225326e-05, 3.981243571615778e-05, 0.00022353240638040006, 0.00012674322351813316, 5.118445187690668e-06, 0.00024116165877785534, 1.2973839147889521e-05, 4.8006963879743125e-06, 2.3321663320530206e-05, 7.723527232883498e-05, 4.59344209957635e-06, 1.9647124645416625e-05, 0.00021129964443389326, 1.1453522347437683e-05, 4.380489826871781e-06, 1.4442671272263397e-05, 3.676743290270679e-05, 0.00011845333210658282, 3.7977195461280644e-05, 0.0007802762556821108, 0.00040483634802512825, 1.042955591401551e-05, 0.6236649751663208, 0.0, 0.0, 0.0, 0.0], [2.0582927390933037e-05, 8.831251761876047e-05, 0.00020454880723264068, 0.0003018901334144175, 8.223887562053278e-05, 1.9704415535670705e-05, 0.00014085126167628914, 2.896317164413631e-05, 7.669385013286956e-06, 3.7246169085847214e-05, 1.673677070357371e-05, 6.404696614481509e-05, 0.0006910591037012637, 1.6027584933908656e-05, 0.00015603694191668183, 0.00014825927792117, 1.5700084986747243e-05, 9.155390580417588e-05, 8.525098382961005e-05, 4.904517481918447e-06, 1.5784396964590997e-05, 5.293117283144966e-05, 0.000529835931956768, 0.0005658396985381842, 6.167324318084866e-05, 6.729497545165941e-05, 0.0003077442815992981, 1.0369159099354874e-05, 6.799342372687533e-05, 0.9960988759994507, 0.0, 0.0, 0.0], [1.3371318345889449e-05, 0.0009821535786613822, 0.0004154810740146786, 0.0001144233756349422, 0.00038730789674445987, 5.660860551870428e-06, 0.0012746280990540981, 0.0005708065000362694, 0.0006383630097843707, 0.0005776663310825825, 4.127733518544119e-06, 9.16175213205861e-06, 9.142841736320406e-05, 3.774867764150258e-06, 1.3575321645475924e-05, 0.0002916179655585438, 3.4474890071578557e-06, 7.899192860350013e-05, 0.003189122537150979, 4.885083853878314e-06, 3.165286670991918e-06, 1.4087469025980681e-05, 0.0001567144354339689, 0.0003544889041222632, 0.00017265471979044378, 0.0013050634879618883, 0.00021867688337806612, 2.6776719096233137e-05, 0.00026460207300260663, 1.3334529285202734e-05, 0.9888005256652832, 0.0, 0.0], [2.745506208157167e-05, 0.00016438262537121773, 7.99642366473563e-05, 0.001191496616229415, 0.0007883630460128188, 2.658414359757444e-06, 3.005756479979027e-05, 7.4579688771336805e-06, 0.00014940995606593788, 2.8857133656856604e-05, 1.873827500276093e-06, 0.00033288003760389984, 5.160855653230101e-05, 1.7577448261363315e-06, 0.0001265659084310755, 0.00014267012011259794, 1.6954454622464254e-06, 2.1952573661110364e-05, 0.00023040804080665112, 4.429338878253475e-05, 1.6103306279546814e-06, 2.7008050892618485e-05, 0.00023884844267740846, 0.00019046018132939935, 9.496136954112444e-06, 0.000446643796749413, 0.00022095769236329943, 5.379170943342615e-06, 0.0006956399301998317, 0.0001547076681163162, 0.0002548544143792242, 0.9943286776542664, 0.0], [0.006231049541383982, 9.72282505244948e-05, 6.871603090985445e-06, 2.1151156033738516e-05, 5.8280591474613175e-05, 0.007238905411213636, 2.0987936295568943e-05, 0.00025459096650592983, 6.243876850930974e-05, 2.0924684577039443e-05, 0.007872147485613823, 5.5853353842394426e-05, 9.868796041700989e-06, 0.009169402532279491, 7.203016866696998e-05, 7.068700597301358e-06, 0.010345976799726486, 0.0013096537441015244, 3.803684376180172e-05, 8.022711699595675e-05, 0.012053261511027813, 4.071998773724772e-05, 3.6860749332845444e-06, 3.4713455534074455e-05, 0.0005061827832832932, 8.918941603042185e-05, 2.9112143238307908e-05, 0.0012772815534844995, 8.489656465826556e-05, 0.00018447409092914313, 0.00013425754150375724, 6.813067739130929e-05, 0.9425214529037476]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.943029522895813, 0.05697042867541313, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9185556173324585, 0.03280005604028702, 0.048644352704286575, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8779287934303284, 0.056434255093336105, 0.04271192103624344, 0.02292500250041485, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.804131805896759, 0.029098201543092728, 0.075567327439785, 0.05643591657280922, 0.03476677089929581, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4943104684352875, 0.02018355205655098, 0.027966560795903206, 0.01831907220184803, 0.03144203498959541, 0.40777823328971863, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6057478785514832, 0.02924242615699768, 0.09491512179374695, 0.07609347254037857, 0.06614658236503601, 0.08705786615610123, 0.0407966710627079, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4483822286128998, 0.04542430862784386, 0.0740148276090622, 0.06864846497774124, 0.09376619011163712, 0.0877426490187645, 0.06534270942211151, 0.11667861044406891, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.49169260263442993, 0.13782073557376862, 0.03955018147826195, 0.06153320148587227, 0.04539964720606804, 0.04073145240545273, 0.06228705495595932, 0.05861866846680641, 0.062366463243961334, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5404124855995178, 0.044426653534173965, 0.03957854211330414, 0.04188809171319008, 0.07529856264591217, 0.04669506475329399, 0.048475231975317, 0.05500520393252373, 0.08293062448501587, 0.02528950944542885, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.28273555636405945, 0.014234175905585289, 0.017647748813033104, 0.011433064006268978, 0.021741706877946854, 0.2666538953781128, 0.015403537079691887, 0.047349266707897186, 0.017767544835805893, 0.013926065526902676, 0.2911074459552765, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.34496626257896423, 0.046116773039102554, 0.05771546810865402, 0.11131857335567474, 0.11289366334676743, 0.027930336073040962, 0.0385919027030468, 0.05656527727842331, 0.05864058807492256, 0.06648595631122589, 0.026114359498023987, 0.052660852670669556, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.46992170810699463, 0.032015684992074966, 0.1077289879322052, 0.027006765827536583, 0.04465881362557411, 0.022773560136556625, 0.0231170691549778, 0.025491517037153244, 0.049502693116664886, 0.026573937386274338, 0.01970885880291462, 0.06337957829236984, 0.08812081813812256, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.20661765336990356, 0.01037929393351078, 0.012261927127838135, 0.008319098502397537, 0.016007069498300552, 0.19527742266654968, 0.011453290469944477, 0.03475669026374817, 0.01307358592748642, 0.010938976891338825, 0.21602483093738556, 0.005866494961082935, 0.023642312735319138, 0.23538129031658173, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.33993491530418396, 0.029161635786294937, 0.09540709853172302, 0.03395187482237816, 0.08440458029508591, 0.012559544295072556, 0.02935866080224514, 0.024564165621995926, 0.10622433573007584, 0.04689216986298561, 0.011469592340290546, 0.006369193084537983, 0.11145274341106415, 0.011317990720272064, 0.0569315105676651, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4040880799293518, 0.024195658043026924, 0.038910072296857834, 0.014727434143424034, 0.024456558749079704, 0.038450006395578384, 0.03923036903142929, 0.037171367555856705, 0.06030004844069481, 0.041985444724559784, 0.037167176604270935, 0.01639123260974884, 0.0392896942794323, 0.03772977367043495, 0.13448578119277954, 0.011421292088925838, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15876448154449463, 0.008203906007111073, 0.0092952661216259, 0.0062217190861701965, 0.011793204583227634, 0.15069791674613953, 0.008851958438754082, 0.026313532143831253, 0.010186923667788506, 0.00843372568488121, 0.16762229800224304, 0.004420033656060696, 0.01821179874241352, 0.183508038520813, 0.020927347242832184, 0.006447790190577507, 0.20010006427764893, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.17539216578006744, 0.02056964486837387, 0.01829143613576889, 0.009298020042479038, 0.017377890646457672, 0.042534783482551575, 0.02070159651339054, 0.050443943589925766, 0.02543804794549942, 0.017218226566910744, 0.04311535507440567, 0.013349834829568863, 0.0285286046564579, 0.045972540974617004, 0.03408820927143097, 0.019834214821457863, 0.04992839694023132, 0.36791715025901794, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.26033347845077515, 0.017148371785879135, 0.03745276480913162, 0.07594798505306244, 0.04674699530005455, 0.018068527802824974, 0.031346458941698074, 0.037415146827697754, 0.07175809890031815, 0.05872536823153496, 0.0170787014067173, 0.04030593857169151, 0.05706355720758438, 0.01711357943713665, 0.1049133837223053, 0.046705346554517746, 0.017230207100510597, 0.024682143703103065, 0.019963975995779037, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.30547064542770386, 0.05190496891736984, 0.043468981981277466, 0.02184685505926609, 0.02101718634366989, 0.03390473872423172, 0.04190470278263092, 0.03909286484122276, 0.028871973976492882, 0.023003432899713516, 0.03205801919102669, 0.023334519937634468, 0.0711059421300888, 0.03290088474750519, 0.061641961336135864, 0.03183262050151825, 0.033767636865377426, 0.04571487754583359, 0.035015225410461426, 0.022141898050904274, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1224295049905777, 0.006105300039052963, 0.006670300383120775, 0.00458177737891674, 0.009337784722447395, 0.11381794512271881, 0.006783359684050083, 0.01971990056335926, 0.00758085772395134, 0.006613647099584341, 0.12765924632549286, 0.0035026604309678078, 0.01400233618915081, 0.14000985026359558, 0.015684885904192924, 0.005092614330351353, 0.15336214005947113, 0.03527415543794632, 0.02246563322842121, 0.006954458076506853, 0.17235167324543, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.20538486540317535, 0.03477850183844566, 0.014682911336421967, 0.03118330053985119, 0.030931269749999046, 0.021952766925096512, 0.03290865570306778, 0.05740531533956528, 0.05587516725063324, 0.04864277318120003, 0.02352074719965458, 0.015108555555343628, 0.02738633006811142, 0.02451845072209835, 0.060604583472013474, 0.034776147454977036, 0.026137804612517357, 0.051684360951185226, 0.06281405687332153, 0.020291464403271675, 0.02860172651708126, 0.0908101499080658, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.23978599905967712, 0.031323499977588654, 0.05037755146622658, 0.01586942747235298, 0.03901456296443939, 0.02280554547905922, 0.04285356029868126, 0.028882304206490517, 0.04046262055635452, 0.0341072604060173, 0.022644521668553352, 0.03923071548342705, 0.0723857581615448, 0.022345518693327904, 0.04958106949925423, 0.03193334490060806, 0.023325597867369652, 0.04521360620856285, 0.030554356053471565, 0.022876497358083725, 0.02473229542374611, 0.055494822561740875, 0.014199569821357727, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.23581692576408386, 0.020558306947350502, 0.04375005513429642, 0.02970486506819725, 0.03703876957297325, 0.014953454956412315, 0.04004311189055443, 0.027184365317225456, 0.0457618422806263, 0.03809259459376335, 0.014181883074343204, 0.03789152577519417, 0.06518243253231049, 0.014182924292981625, 0.05489495024085045, 0.023720961064100266, 0.014592787250876427, 0.025570029392838478, 0.07356183975934982, 0.039182331413030624, 0.014925522729754448, 0.04628865420818329, 0.027801064774394035, 0.015118853189051151, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14254964888095856, 0.012010160833597183, 0.016881300136446953, 0.020742561668157578, 0.032451752573251724, 0.029625998809933662, 0.030295221135020256, 0.05620869994163513, 0.029608748853206635, 0.02948184125125408, 0.03263983502984047, 0.010038234293460846, 0.04078620672225952, 0.03462785482406616, 0.03391636535525322, 0.02015574835240841, 0.036843158304691315, 0.0606469102203846, 0.047447122633457184, 0.032525286078453064, 0.040400370955467224, 0.05947763845324516, 0.03129400685429573, 0.04792547971010208, 0.07141980528831482, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2222907692193985, 0.059111371636390686, 0.03702675551176071, 0.04059014096856117, 0.0272544976323843, 0.019174763932824135, 0.03171538934111595, 0.02046213671565056, 0.038113273680210114, 0.019927890971302986, 0.018538912758231163, 0.015436704270541668, 0.04536491632461548, 0.01935577020049095, 0.050358302891254425, 0.03328138589859009, 0.02017974853515625, 0.03679436072707176, 0.04331387206912041, 0.028476405888795853, 0.02131732925772667, 0.047712888568639755, 0.013107037171721458, 0.02633604407310486, 0.030211349949240685, 0.03454800695180893, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.28438133001327515, 0.023870104923844337, 0.04641882702708244, 0.010260095819830894, 0.03390985727310181, 0.01830146834254265, 0.0237088892608881, 0.0247668344527483, 0.025767752900719643, 0.022968804463744164, 0.016735462471842766, 0.013406947255134583, 0.045986607670784, 0.01667322963476181, 0.08106391876935959, 0.05033260956406593, 0.016726160421967506, 0.019904790446162224, 0.03032534383237362, 0.01014632172882557, 0.01731823943555355, 0.019040953367948532, 0.01108111348003149, 0.052046384662389755, 0.033353518694639206, 0.03880883380770683, 0.012695519253611565, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.11854357272386551, 0.01335094589740038, 0.013422034680843353, 0.030273202806711197, 0.02616293355822563, 0.020776210352778435, 0.021280204877257347, 0.03798932954668999, 0.03536444157361984, 0.036651790142059326, 0.022004006430506706, 0.01538003608584404, 0.030023131519556046, 0.023299671709537506, 0.03000757284462452, 0.014861369505524635, 0.02452193759381771, 0.03999800235033035, 0.03900156170129776, 0.036007679998874664, 0.026851212605834007, 0.06795871257781982, 0.038931310176849365, 0.057258062064647675, 0.0665983259677887, 0.03075685352087021, 0.023343220353126526, 0.059382691979408264, 0.0, 0.0, 0.0, 0.0, 0.0], [0.25091853737831116, 0.01135188713669777, 0.023033952340483665, 0.01884976588189602, 0.013208821415901184, 0.016430404037237167, 0.03763698413968086, 0.021519413217902184, 0.03823148459196091, 0.03122309409081936, 0.01608109660446644, 0.017179012298583984, 0.0823964774608612, 0.01577541045844555, 0.04969313368201256, 0.033979032188653946, 0.016469145193696022, 0.025656109675765038, 0.053269319236278534, 0.024380972608923912, 0.017032906413078308, 0.031062550842761993, 0.014834268018603325, 0.04310780018568039, 0.027818351984024048, 0.015127941034734249, 0.012749841436743736, 0.026852674782276154, 0.014129597693681717, 0.0, 0.0, 0.0, 0.0], [0.19256946444511414, 0.022833675146102905, 0.01449588406831026, 0.028055010363459587, 0.03290426358580589, 0.018577100709080696, 0.023776723071932793, 0.014988909475505352, 0.027755476534366608, 0.01995212957262993, 0.018426887691020966, 0.0268084853887558, 0.04026195779442787, 0.01895732805132866, 0.01998024806380272, 0.039050325751304626, 0.01949433982372284, 0.030714301392436028, 0.07932323962450027, 0.03619766980409622, 0.02037958987057209, 0.023319289088249207, 0.018723847344517708, 0.05692768841981888, 0.023927535861730576, 0.03924323245882988, 0.02178351581096649, 0.020374197512865067, 0.04178833216428757, 0.00840924121439457, 0.0, 0.0, 0.0], [0.1891939342021942, 0.01435206737369299, 0.027829350903630257, 0.0189360398799181, 0.05455230921506882, 0.024302387610077858, 0.0207525584846735, 0.03050178475677967, 0.01690041646361351, 0.029904041439294815, 0.023111775517463684, 0.02166486158967018, 0.03335980698466301, 0.023050500079989433, 0.02792702615261078, 0.026253484189510345, 0.0242579597979784, 0.024394970387220383, 0.017666669562458992, 0.02208106406033039, 0.025441503152251244, 0.023996729403734207, 0.015941407531499863, 0.021863479167222977, 0.04876275360584259, 0.01023405697196722, 0.025762980803847313, 0.06192327290773392, 0.06737110018730164, 0.01818411611020565, 0.009525515139102936, 0.0, 0.0], [0.27111950516700745, 0.05460572615265846, 0.039705973118543625, 0.03512895852327347, 0.02031376212835312, 0.0081838583573699, 0.022556638345122337, 0.01403643935918808, 0.02619846910238266, 0.032355356961488724, 0.007185023743659258, 0.010813490487635136, 0.05162610113620758, 0.007057540584355593, 0.04232776537537575, 0.01850598305463791, 0.006952997762709856, 0.012550847604870796, 0.037855181843042374, 0.014787373133003712, 0.007130768615752459, 0.01895672082901001, 0.01075897179543972, 0.023540528491139412, 0.01625620573759079, 0.007915404625236988, 0.03652312234044075, 0.014562270604074001, 0.019251469522714615, 0.010911807417869568, 0.0343967042863369, 0.06592899560928345, 0.0], [0.14836537837982178, 0.014506888575851917, 0.007646295242011547, 0.011038758791983128, 0.02630247175693512, 0.01319483295083046, 0.027184898033738136, 0.017392203211784363, 0.016122139990329742, 0.032880790531635284, 0.014050280675292015, 0.0071673463098704815, 0.016120055690407753, 0.014647023752331734, 0.018450895324349403, 0.013102126307785511, 0.01528322696685791, 0.01454298384487629, 0.05961218476295471, 0.02238387055695057, 0.016796991229057312, 0.10928259789943695, 0.07264000922441483, 0.08393709361553192, 0.025248989462852478, 0.03774847462773323, 0.016226941719651222, 0.024953871965408325, 0.042165856808423996, 0.004525808617472649, 0.014491353183984756, 0.01914571039378643, 0.022841647267341614]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09646990895271301, 0.9035300612449646, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04325238987803459, 0.08177754282951355, 0.8749701380729675, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09995394200086594, 0.025312727317214012, 0.02010803483426571, 0.8546252846717834, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.024889368563890457, 0.00320735527202487, 0.0018421593122184277, 0.022361503913998604, 0.9476996660232544, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10732383280992508, 0.017841672524809837, 0.01955333538353443, 0.04333319514989853, 0.10211498290300369, 0.7098329067230225, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.006426361855119467, 0.00044797800364904106, 0.00014756589371245354, 0.00046936701983213425, 0.0014411886222660542, 0.003859696676954627, 0.98720782995224, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00103545852471143, 0.00019901820633094758, 0.00016020411567296833, 6.937271973583847e-05, 0.00038674141978845, 0.005171590484678745, 0.8964057564735413, 0.0965719074010849, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0012883321614935994, 0.0003233453317079693, 0.00026527189766056836, 0.00025490616098977625, 0.000201298258616589, 0.00010049015691038221, 0.0005700886249542236, 0.000409130152547732, 0.9965871572494507, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0001874157169368118, 1.6196530850720592e-05, 4.2281039895897266e-06, 0.000287588540231809, 1.1125704986625351e-05, 9.805656191019807e-06, 0.0001556719362270087, 7.632971392013133e-05, 0.0034869094379246235, 0.9957647323608398, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.015741512179374695, 0.0006393603980541229, 0.00045715278247371316, 0.0009912519017234445, 0.0021140354219824076, 0.0180897768586874, 0.0471203438937664, 0.07010912150144577, 0.06152832508087158, 0.27690398693084717, 0.5063051581382751, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0006166594685055315, 0.0005207078065723181, 4.61151976196561e-05, 0.00146130018401891, 0.0005623754695989192, 4.4476037146523595e-05, 0.00036539664142765105, 0.0002860166132450104, 0.004506127443164587, 0.005816521123051643, 0.0007244544103741646, 0.9850499033927917, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0010478557087481022, 3.062814721488394e-05, 0.00017344979278277606, 5.423119000624865e-05, 6.388417386915535e-05, 1.1261126928729936e-05, 1.7169008060591295e-05, 1.3931307876191568e-05, 0.0020760390907526016, 0.00026926834834739566, 0.00015268517017830163, 0.0036844322457909584, 0.992405116558075, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.010746272280812263, 0.00025354334502480924, 0.0001532292371848598, 0.00029287015786394477, 0.0005376793560571969, 0.004423544742166996, 0.00988433975726366, 0.012843050062656403, 0.012738612480461597, 0.05966852977871895, 0.107729971408844, 0.024745700880885124, 0.07808694988489151, 0.6778956651687622, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00041218000114895403, 2.6218738639727235e-05, 1.7755904991645366e-05, 0.00019181883544661105, 3.2979492061713245e-06, 3.912682132067857e-06, 1.043809788825456e-05, 4.9068494263337925e-06, 0.0005868570297025144, 0.003038151189684868, 3.693124745041132e-05, 0.0007724304450675845, 0.009622604586184025, 0.00016094396414700896, 0.9851114153862, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00010886583186220378, 3.5845487218466587e-06, 5.191657237446634e-06, 3.083837145823054e-05, 1.8408762116450816e-05, 7.954765806061914e-07, 3.3532073757669423e-06, 6.574365215783473e-06, 0.0007270933128893375, 0.0018232465954497457, 7.853259376133792e-06, 0.0003062748000957072, 0.006975048687309027, 3.4850869269575924e-05, 0.0282927006483078, 0.9616552591323853, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.006073986645787954, 8.815194451017305e-05, 4.5837816287530586e-05, 8.250484825111926e-05, 0.0001272043155040592, 0.0009312513284385204, 0.0016699967673048377, 0.001938193803653121, 0.002101697726175189, 0.010131671093404293, 0.016047311946749687, 0.003835388459265232, 0.012242135591804981, 0.10037057846784592, 0.07853133976459503, 0.10997355729341507, 0.655809223651886, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0021862962748855352, 1.1892905604327098e-05, 1.673595943429973e-05, 2.081541060761083e-05, 2.116251016559545e-05, 0.000771492428611964, 0.0008652104297652841, 0.0005560291465371847, 0.0001565588463563472, 0.0019499900517985225, 0.010891195386648178, 0.0003954307467211038, 0.0015144682256504893, 0.06916307657957077, 0.00430810172110796, 0.004035161342471838, 0.4939703941345215, 0.40916600823402405, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [1.979802800633479e-05, 4.972388865098765e-07, 9.16571263331889e-09, 1.3230416584519844e-07, 5.6018219396491986e-08, 1.1903876995233986e-08, 6.153105118755775e-07, 4.0685506519366754e-08, 4.517455181485275e-06, 1.4017791727383155e-05, 1.6456915830076468e-07, 1.086384372683824e-06, 7.44406133890152e-06, 1.005667400022503e-06, 2.75950224022381e-05, 0.00031378038693219423, 6.201125415827846e-06, 8.213370165321976e-06, 0.9995948672294617, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [6.497167760244338e-06, 8.692019264344708e-07, 1.4051378229851252e-06, 3.7704828059759166e-07, 3.3291723866568645e-07, 6.104961869368708e-08, 2.542995503063139e-08, 1.1067748317827864e-07, 3.670873775263317e-05, 5.704808927475824e-07, 6.172335247356386e-07, 0.00015627949323970824, 0.00014737885794602334, 3.010375849044067e-06, 6.130654219305143e-05, 0.0017769918777048588, 1.8545919374446385e-05, 3.426316106924787e-05, 0.923351526260376, 0.07440318167209625, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.003649001009762287, 3.7494322896236554e-05, 1.74708020495018e-05, 2.8449197998270392e-05, 3.312424087198451e-05, 0.00019569217693060637, 0.0002647593501023948, 0.0002469043538440019, 0.00028033158741891384, 0.0013949184212833643, 0.0017910305177792907, 0.00042468024184927344, 0.001462370972149074, 0.009717755019664764, 0.008053947240114212, 0.01320058386772871, 0.06315966695547104, 0.07152792811393738, 0.07523112744092941, 0.024205094203352928, 0.7250776290893555, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.000641089805867523, 3.925701548723737e-06, 1.1607372698563267e-06, 1.537548200758465e-06, 9.072650755115319e-07, 8.05819763627369e-06, 1.3223971109255217e-05, 4.6199284042813815e-06, 3.678836583276279e-05, 4.5651795517187566e-05, 7.577823998872191e-05, 2.274175494676456e-05, 5.6684530136408284e-05, 0.0004737511626444757, 0.0003640766954049468, 0.00042400247184559703, 0.00375733501277864, 0.004363126587122679, 0.007395228371024132, 0.005022629629820585, 0.05575620383024216, 0.9215314984321594, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0005377684719860554, 8.12255109394755e-07, 2.3018603769742185e-06, 1.470038341722102e-06, 1.8703678961173864e-06, 6.02451621034561e-07, 4.12783674619277e-06, 1.6705761254343088e-06, 2.047191856036079e-06, 7.430045661749318e-05, 3.5942291560786543e-06, 7.171521247073542e-06, 3.052820102311671e-05, 1.8927734345197678e-05, 0.0002543879672884941, 0.0001949639990925789, 0.00012626573152374476, 0.00018230534624308348, 0.0037352300714701414, 0.0006727128056809306, 0.0015798343811184168, 0.011177700012922287, 0.9813893437385559, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [8.512814383720979e-05, 4.870869929618493e-07, 1.2458942819648655e-06, 2.0811235401652084e-07, 1.92428601053507e-07, 3.704030859807972e-08, 4.5552428673545364e-07, 5.504545086409962e-08, 1.532036321805208e-06, 5.083304131403565e-06, 2.1400977345820138e-07, 1.459531858927221e-06, 1.4768386790819932e-05, 1.0635022817950812e-06, 2.9461503800121136e-05, 5.7288212701678276e-05, 6.615699021494947e-06, 7.844223546271678e-06, 0.00015063839964568615, 0.00022636978246737272, 7.973578613018617e-05, 0.00013040869089309126, 0.010868428274989128, 0.988331139087677, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0004166325961705297, 2.250373881906853e-06, 5.27054771737312e-06, 2.7549118385650218e-05, 3.963056951761246e-05, 7.514749540860066e-06, 8.10094024927821e-06, 1.1581340913835447e-05, 1.320597675658064e-05, 6.750728789484128e-05, 2.5212739274138585e-05, 1.486069868406048e-05, 7.521701627410948e-05, 0.00011457462824182585, 0.0002574531245045364, 0.0003666863194666803, 0.0006904753972776234, 0.0014518487732857466, 0.0032460209913551807, 0.0006353403441607952, 0.008745575323700905, 0.015190862119197845, 0.01815020479261875, 0.1330331563949585, 0.8174033164978027, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [1.673920633038506e-05, 8.245287830277448e-08, 3.257373748510872e-08, 2.873728810470766e-08, 1.6217634879467369e-07, 1.5275355425004022e-09, 2.3278557748085404e-08, 5.576718198341268e-09, 1.2825742601307866e-07, 1.3429436762635305e-07, 6.160537857624604e-09, 2.5232478151338e-07, 3.2057448606792605e-06, 2.6277179543399143e-08, 7.619933626301645e-07, 7.901945537014399e-06, 1.3940400833689637e-07, 1.955255868324457e-07, 0.00032022525556385517, 8.173647074727342e-07, 1.562996089887747e-06, 3.763153017644072e-06, 0.0007450500270351768, 0.0037152147851884365, 3.934417327400297e-05, 0.9951443076133728, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00023842290102038532, 5.040124960942194e-06, 1.3208604059400386e-06, 8.599836291978136e-06, 5.166078608453972e-06, 1.3470925352976337e-07, 6.710816364829952e-07, 2.699698029573483e-07, 6.345731435430935e-06, 4.417831223690882e-05, 2.7444934858067427e-07, 2.017119368247222e-05, 1.652665923757013e-05, 8.122791541609331e-07, 0.0001077100241673179, 0.0001239744306076318, 3.5075922824034933e-06, 8.505840924044605e-06, 6.980268517509103e-05, 0.0002681369078345597, 2.8792866942239925e-05, 0.00011957906826864928, 0.0001296169066336006, 0.007425930816680193, 0.0007109709549695253, 0.005815016105771065, 0.9848405718803406, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00040899505256675184, 6.095365279179532e-07, 1.9963911199738504e-06, 9.31032991502434e-06, 3.769046315937885e-06, 1.987656105484348e-06, 4.57654266483587e-07, 1.069074301085493e-06, 3.087174036409124e-06, 9.211397809849586e-06, 4.724369318864774e-06, 1.2070032653355156e-06, 5.095620508654974e-06, 1.8047117919195443e-05, 4.857865133089945e-05, 4.169386738794856e-05, 0.00010537513298913836, 0.00022466867812909186, 0.00011528613686095923, 0.0002492894127499312, 0.0013673142530024052, 0.0016190343303605914, 0.0013275155797600746, 0.003491780487820506, 0.06305663287639618, 0.00646906066685915, 0.0951264277100563, 0.8262877464294434, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00031804729951545596, 1.4181842971083825e-06, 3.1855009297032666e-07, 2.7823105028801365e-06, 5.388588397181593e-05, 4.300602540752152e-08, 5.386270913732005e-07, 8.497841008647811e-08, 5.99971485826245e-07, 4.110329882678343e-06, 7.099399113030813e-08, 8.454308044747449e-07, 2.5266151624236954e-06, 2.198386539475905e-07, 6.098252015362959e-06, 2.1130046661710367e-05, 9.411802466274821e-07, 9.95040522866475e-07, 3.041357376787346e-05, 2.976995119752246e-06, 8.453868758806493e-06, 2.6007013730122708e-05, 0.00023055852216202766, 0.0013383383629843593, 0.0002175826084567234, 0.007014401722699404, 0.017924025654792786, 0.0005302676581777632, 0.9722622632980347, 0.0, 0.0, 0.0, 0.0], [1.8505332377571904e-07, 5.565692351439111e-09, 7.262683787701008e-10, 7.745998509278706e-09, 1.507951097323712e-08, 8.22852175286215e-10, 1.7915348093922034e-09, 1.7345647140842857e-09, 1.0962556196147943e-08, 5.0032749498996054e-08, 3.9228162940219136e-09, 1.0013924622853665e-07, 2.333774595797422e-08, 1.6555254944705666e-08, 5.1713197279923406e-08, 1.614605764643784e-07, 9.827901692460728e-08, 2.707449482386437e-07, 5.411987331171986e-06, 2.5083886612264905e-07, 1.1805956319221877e-06, 2.8004205887555145e-06, 4.61038098364952e-06, 0.00022679210815113038, 5.7942423154599965e-05, 0.0004519550711847842, 0.0001197646779473871, 0.00023980921832844615, 0.001874006469734013, 0.9970145225524902, 0.0, 0.0, 0.0], [4.347462163423188e-05, 2.968874923681142e-06, 3.919368793958711e-07, 9.117065360442211e-07, 4.364489711861097e-07, 4.686591115188321e-09, 9.240378062713717e-08, 1.2419349992853768e-08, 7.188964445958845e-08, 6.304543944679608e-07, 6.108888950251412e-09, 1.9987160726486763e-07, 1.4578674836229766e-06, 1.4727343788933922e-08, 1.7957326292616926e-07, 2.6422712835483253e-05, 5.112656609185251e-08, 4.710674161856332e-08, 0.0015761180547997355, 2.198941047026892e-06, 3.3339856031489035e-07, 2.1814876163261943e-06, 0.00016240392869804054, 0.0017777654575183988, 1.1749207260436378e-05, 0.0036160903982818127, 6.199482595548034e-05, 3.5168231988791376e-05, 0.001148868934251368, 0.016085581853985786, 0.9754422307014465, 0.0, 0.0], [0.0011562927393242717, 6.199047675181646e-06, 6.137794912319805e-07, 3.102660230069887e-06, 8.228408319155278e-07, 4.3660193682626414e-08, 9.36325577072239e-08, 2.7735543639550997e-08, 5.109529865876539e-07, 1.2355309308986762e-06, 3.8253045175906664e-08, 1.127230689235148e-06, 1.9008897425010218e-06, 8.714057031511402e-08, 5.398183020588476e-06, 7.393444320769049e-06, 3.128881189695676e-07, 3.0146313179102435e-07, 0.00013595378550235182, 1.3343835234991275e-05, 2.39964992942987e-06, 3.871273293043487e-05, 0.0008219636511057615, 0.0006131280679255724, 2.151841727027204e-05, 0.004140730947256088, 0.0008760862983763218, 0.00011684626952046528, 0.0021590692922472954, 0.027525268495082855, 0.04434971883893013, 0.9179997444152832, 0.0], [0.003194290678948164, 4.681676728068851e-05, 3.49913134414237e-05, 2.7043113732361235e-05, 1.4000337614561431e-05, 1.7085165382013656e-05, 8.051893928495701e-06, 3.909831775672501e-06, 1.0716959877754562e-05, 1.793081537471153e-05, 1.4207512322172988e-05, 2.691133522603195e-05, 2.6411998987896368e-05, 3.1100229534786195e-05, 1.1198581887583714e-05, 5.485505607794039e-05, 0.00011474490747787058, 0.00010493675654288381, 0.00028063071658834815, 0.00020839076023548841, 0.0009329256135970354, 0.0018990016542375088, 0.0011267587542533875, 0.0019257268868386745, 0.0039171562530100346, 0.007499460596591234, 0.010695938020944595, 0.023199880495667458, 0.03039817325770855, 0.12729351222515106, 0.04325327277183533, 0.1766805499792099, 0.5669293999671936]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.25308629870414734, 0.746913731098175, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.30671578645706177, 0.32906386256217957, 0.3642203211784363, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.07416975498199463, 0.1618966907262802, 0.054325949400663376, 0.7096075415611267, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1668822318315506, 0.039017826318740845, 0.03822460398077965, 0.21398353576660156, 0.5418918132781982, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.19483636319637299, 0.2111925631761551, 0.05150565132498741, 0.0870383232831955, 0.2299954891204834, 0.22543159127235413, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.13900135457515717, 0.02974863536655903, 0.03860695660114288, 0.05133272334933281, 0.19284246861934662, 0.08012373745441437, 0.4683440625667572, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08969778567552567, 0.04080599546432495, 0.03473307564854622, 0.08414525538682938, 0.09911047667264938, 0.07059449702501297, 0.1361657828092575, 0.4447471499443054, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05986519902944565, 0.020190449431538582, 0.018785323947668076, 0.1058472990989685, 0.057948824018239975, 0.027517711743712425, 0.0566631518304348, 0.08226760476827621, 0.5709145069122314, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.010379260405898094, 0.0046982537023723125, 0.004143984522670507, 0.0072914063930511475, 0.006256693042814732, 0.0033180469181388617, 0.009175012819468975, 0.018754413351416588, 0.03393147885799408, 0.9020513892173767, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.07270114123821259, 0.0477648489177227, 0.010500345379114151, 0.0161594245582819, 0.046178195625543594, 0.05249456688761711, 0.051845718175172806, 0.21189512312412262, 0.035158202052116394, 0.17267179489135742, 0.2826306223869324, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.015126891434192657, 0.004731189925223589, 0.0023226316552609205, 0.0065757399424910545, 0.01836245507001877, 0.003339666873216629, 0.008784224279224873, 0.007409745827317238, 0.006289808079600334, 0.07638943940401077, 0.01200348511338234, 0.838664710521698, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.03233193978667259, 0.010628965683281422, 0.0026151640340685844, 0.0011762931244447827, 0.0030932866502553225, 0.0015055524418130517, 0.007079718168824911, 0.00283446884714067, 0.005003884434700012, 0.012032588012516499, 0.0039877742528915405, 0.051872409880161285, 0.8658380508422852, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05045590549707413, 0.02653883770108223, 0.005783271975815296, 0.00808729324489832, 0.021799379959702492, 0.025670334696769714, 0.022779036313295364, 0.09139905869960785, 0.01501737255603075, 0.07093921303749084, 0.12448635697364807, 0.1266833245754242, 0.0610213503241539, 0.34933918714523315, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05474122613668442, 0.05296846479177475, 0.00398812722414732, 0.012351620942354202, 0.00441562058404088, 0.0035962786059826612, 0.011385707184672356, 0.009828065522015095, 0.014749741181731224, 0.07078825682401657, 0.011209187097847462, 0.05164055898785591, 0.1973438560962677, 0.027129262685775757, 0.4738638997077942, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.007163074798882008, 0.004963848739862442, 0.0027692278381437063, 0.0019424431957304478, 0.010544748045504093, 0.0014144877204671502, 0.003663665847852826, 0.003149948548525572, 0.005481299012899399, 0.021614043042063713, 0.003922200761735439, 0.07935082912445068, 0.2247791737318039, 0.009166955016553402, 0.027402665466070175, 0.5926714539527893, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.034611064940690994, 0.015296351164579391, 0.003095126012340188, 0.0038990958128124475, 0.009989401325583458, 0.01144686620682478, 0.009189358912408352, 0.03356517106294632, 0.005761212203651667, 0.025581011548638344, 0.043846867978572845, 0.04521361365914345, 0.02055339701473713, 0.1196233481168747, 0.1567324846982956, 0.13173452019691467, 0.32986119389533997, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.017253872007131577, 0.004397525917738676, 0.004968690220266581, 0.007064312230795622, 0.006634894758462906, 0.006910949479788542, 0.011386900208890438, 0.016178512945771217, 0.020317763090133667, 0.020396802574396133, 0.02342209964990616, 0.01890912838280201, 0.04024204611778259, 0.06200675293803215, 0.116453617811203, 0.07975102961063385, 0.16864468157291412, 0.37506037950515747, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.002459116280078888, 0.00024919764837250113, 3.400466084713116e-05, 0.0002113294176524505, 0.00020066798606421798, 0.0001258924457943067, 0.00036506837932392955, 0.00037359801353886724, 0.00013798549480270594, 0.0004702212172560394, 0.00027151827816851437, 0.003417074913159013, 0.0006049809162504971, 0.0005454609636217356, 0.0008937679813243449, 0.0015852133510634303, 0.0012150758411735296, 0.00282126828096807, 0.9840186238288879, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00015282434469554573, 0.038923632353544235, 0.00047151994658634067, 0.0003731817996595055, 3.271793684689328e-05, 1.063560375769157e-05, 3.8245766518230084e-06, 1.3244694855529815e-05, 6.418924022000283e-05, 2.7344345653546043e-05, 2.8319956982159056e-05, 0.0022295680828392506, 0.0013866383815184236, 7.163518603192642e-05, 0.0001402555499225855, 0.00824644137173891, 0.00019678636454045773, 0.00020546668383758515, 0.8976381421089172, 0.049783725291490555, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.02645810693502426, 0.010410889983177185, 0.0021332555916160345, 0.0024560203310102224, 0.005542597267776728, 0.005954570136964321, 0.0041958331130445, 0.013451533392071724, 0.0023519538808614016, 0.009471958503127098, 0.015433433465659618, 0.014804964885115623, 0.006865760777145624, 0.037873148918151855, 0.047449350357055664, 0.045472823083400726, 0.10199166089296341, 0.08666391670703888, 0.06470184028148651, 0.10160762816667557, 0.3947088122367859, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.021157892420887947, 0.0035484335385262966, 0.001744111767038703, 0.0035093079786747694, 0.004201894626021385, 0.00257777888327837, 0.004015856888145208, 0.004598999861627817, 0.003955156076699495, 0.006455833092331886, 0.00590789457783103, 0.0037331636995077133, 0.009013411588966846, 0.013927980326116085, 0.029895801097154617, 0.02008231170475483, 0.03804440051317215, 0.06883092224597931, 0.0815289318561554, 0.0383402481675148, 0.15160253643989563, 0.48332712054252625, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0014211690286174417, 0.00040690117748454213, 0.0002714421134442091, 0.0014869242440909147, 0.0006475155241787434, 0.0002567152841947973, 0.00027297638007439673, 0.0005059852264821529, 0.00017537492385599762, 0.0012059591244906187, 0.0005902894772589207, 0.00043073901906609535, 0.00036999606527388096, 0.0013322837185114622, 0.000759427435696125, 0.002518307650461793, 0.003599978983402252, 0.004756716545671225, 0.011892932467162609, 0.0034102171193808317, 0.013851108960807323, 0.05907328426837921, 0.8907637596130371, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00505886459723115, 0.0008823543903417885, 0.0015358274104073644, 0.0016909865662455559, 0.0013438736787065864, 0.0004949708236381412, 0.0006572254933416843, 0.000552159093786031, 0.0005812101298943162, 0.002054560463875532, 0.0008380875224247575, 0.001006747712381184, 0.0011300853220745921, 0.0016139923827722669, 0.005022874567657709, 0.014576874673366547, 0.0037143337540328503, 0.004681476857513189, 0.007913822308182716, 0.006778388284146786, 0.012291365303099155, 0.027733024209737778, 0.10214646905660629, 0.7957004308700562, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00400469871237874, 0.0004576975479722023, 0.0015645211096853018, 0.0031372052617371082, 0.001993746031075716, 0.00037892640102654696, 0.00049577810568735, 0.000569801835808903, 0.001005206722766161, 0.0023432248272001743, 0.0005772155127488077, 0.0016756145050749183, 0.003078736364841461, 0.0012285250704735518, 0.005278328433632851, 0.00632805610075593, 0.003151731565594673, 0.0053193108178675175, 0.0213831327855587, 0.006147712003439665, 0.012332223355770111, 0.07402164489030838, 0.1211402490735054, 0.561111569404602, 0.16127507388591766, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0011286542285233736, 4.9889109504874796e-05, 4.129114677198231e-05, 9.24961204873398e-05, 5.513622454600409e-05, 3.8617254176642746e-05, 5.5920278100529686e-05, 0.00014896267384756356, 5.509220500243828e-06, 6.817427492933348e-05, 3.6204455682309344e-05, 4.623961285687983e-05, 0.001065501943230629, 5.91987554798834e-05, 7.787253707647324e-05, 0.00038421223871409893, 0.00011692437692545354, 0.0002242824120912701, 0.000508398690726608, 0.0001562813704367727, 0.00032331488910131156, 0.0007086826371960342, 0.0009207415860146284, 0.0008559715934097767, 0.004414450377225876, 0.9884170889854431, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0024299705401062965, 0.003659637412056327, 0.0006344981375150383, 0.0006202656077221036, 0.00298767676576972, 0.00034763681469485164, 0.0005059166578575969, 0.00041149763274006546, 0.00021321582607924938, 0.0006509345257654786, 0.0003955696302000433, 0.16986453533172607, 0.002188899554312229, 0.0006812380161136389, 0.000868376693688333, 0.008905136026442051, 0.0014638527063652873, 0.0013394575798884034, 0.006495129782706499, 0.007638717070221901, 0.004619399085640907, 0.003161790780723095, 0.014200896956026554, 0.046396613121032715, 0.02278606779873371, 0.21939247846603394, 0.4771406054496765, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.015610319562256336, 0.0009648068808019161, 0.0010281825670972466, 0.007148513570427895, 0.005316650029271841, 0.0005432680482044816, 0.0007839886820875108, 0.0004708298947662115, 0.0022980361245572567, 0.005463092587888241, 0.0005483070272020996, 0.0008661497267894447, 0.0009035550756379962, 0.0009251966839656234, 0.008375934325158596, 0.0014478107914328575, 0.0020614899694919586, 0.003693382255733013, 0.017008859664201736, 0.006138675846159458, 0.007041892036795616, 0.04128709062933922, 0.06922487169504166, 0.26617181301116943, 0.05024212971329689, 0.11445137113332748, 0.13764896988868713, 0.23233483731746674, 0.0, 0.0, 0.0, 0.0, 0.0], [0.021895311772823334, 0.0013514558086171746, 0.0013727976474910975, 0.005157721694558859, 0.01093280129134655, 0.00044522207463160157, 0.0016178301302716136, 0.0005713265272788703, 0.0007700274582020938, 0.003093535080552101, 0.0003616345056798309, 0.0009329011081717908, 0.001903240685351193, 0.0005254389252513647, 0.0007036188035272062, 0.0015780640533193946, 0.0010200472315773368, 0.0009346723672933877, 0.005167578347027302, 0.0007855422445572913, 0.0028796829283237457, 0.006337142549455166, 0.04951414093375206, 0.021886898204684258, 0.03004683181643486, 0.06941598653793335, 0.02995387092232704, 0.061116255819797516, 0.6677284240722656, 0.0, 0.0, 0.0, 0.0], [0.0002946656895801425, 8.08511977083981e-05, 0.00011992631334578618, 0.00017154013039544225, 0.0003899005532730371, 8.630252705188468e-05, 5.253727795206942e-05, 6.616376049350947e-05, 0.0001137837243732065, 6.200082134455442e-05, 0.00011238027218496427, 0.0005867315921932459, 9.785001748241484e-05, 0.00021334877237677574, 0.00040444283513352275, 0.0004726238548755646, 0.0004730736545752734, 0.0005199519800953567, 0.0008571160142309964, 0.00022720196284353733, 0.0015976768918335438, 0.0014780627097934484, 0.0016501408535987139, 0.009339644573628902, 0.01279268879443407, 0.012606951408088207, 0.021659579128026962, 0.02886745147407055, 0.07784783840179443, 0.8267576694488525, 0.0, 0.0, 0.0], [0.0006396545795723796, 0.00019920240447390825, 0.0003378460241947323, 0.00032286165514960885, 0.00038839809712953866, 2.2511459974339232e-05, 0.00011559655831661075, 1.968370270333253e-05, 8.749768312554806e-05, 0.00016675743972882628, 2.1353174815885723e-05, 0.00018257695774082094, 0.002879622159525752, 3.2708376238588244e-05, 6.355983350658789e-05, 0.00090785016072914, 6.31420552963391e-05, 4.9545076763024554e-05, 0.005010190419852734, 0.00021555765124503523, 0.00018923325114883482, 0.00023579830303788185, 0.0021191714331507683, 0.0034752420615404844, 0.0004957506898790598, 0.012673038989305496, 0.0026274591218680143, 0.0011926377192139626, 0.028011513873934746, 0.1421458125114441, 0.7951081991195679, 0.0, 0.0], [0.005492142401635647, 0.0014179605059325695, 6.310038588708267e-05, 0.0009282368700951338, 0.0004675872914958745, 9.701211820356548e-05, 5.678690649801865e-05, 6.824053707532585e-05, 4.054806777276099e-05, 0.00021011428907513618, 6.534699787152931e-05, 0.00011957778770010918, 9.082323231268674e-05, 9.077344293473288e-05, 0.00013827496150042862, 0.00041228361078538, 0.00016900571063160896, 0.00013570708688348532, 0.002428320934996009, 0.00020654806576203555, 0.0004959595971740782, 0.00016561975644435734, 0.001825400977395475, 0.001680815708823502, 0.001018735347315669, 0.01507147029042244, 0.003408371703699231, 0.003384194802492857, 0.01886550709605217, 0.010037437081336975, 0.008259555324912071, 0.923088550567627, 0.0], [0.009386110119521618, 0.0017380556091666222, 0.0018091682577505708, 0.0014350252458825707, 0.0016812816029414535, 0.001979130320250988, 0.0014143437147140503, 0.0021127830259501934, 0.0012016963446512818, 0.0010467646643519402, 0.001253248774446547, 0.0012125695357099175, 0.0006952421390451491, 0.0016850176034495234, 0.001290497719310224, 0.0014949428150430322, 0.003052463522180915, 0.006176403723657131, 0.003993155900388956, 0.002994521288201213, 0.008827597834169865, 0.009542387910187244, 0.010477354750037193, 0.0214514322578907, 0.062213558703660965, 0.026861043646931648, 0.033906422555446625, 0.11987609416246414, 0.05027468875050545, 0.026796849444508553, 0.031165700405836105, 0.10728635638952255, 0.4436681270599365]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10520488768815994, 0.8947951793670654, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.03907204046845436, 0.0020172216463834047, 0.9589106440544128, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.015578641556203365, 0.0008392541203647852, 0.0006979768513701856, 0.9828841090202332, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.008856471627950668, 9.934287845680956e-06, 1.1174832252436318e-05, 0.000302469969028607, 0.9908198714256287, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.35445019602775574, 0.030589278787374496, 0.059889569878578186, 0.022903528064489365, 0.0474759079515934, 0.4846915602684021, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04476895555853844, 0.001566466293297708, 0.00037738497485406697, 0.0002507324970792979, 0.00040889676893129945, 0.0002606561465654522, 0.9523669481277466, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09058346599340439, 0.0011470747413113713, 0.006027346942573786, 0.000546847702935338, 0.0017094232607632875, 0.00378508516587317, 0.002684543142095208, 0.8935161828994751, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0060107228346168995, 2.4289069187943824e-05, 0.00031266745645552874, 1.8682934751268476e-05, 0.0002979248820338398, 9.904525541060138e-06, 6.619561190746026e-06, 7.912206569926639e-07, 0.9933184385299683, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.004546814598143101, 0.0002001934335567057, 0.00029920213273726404, 0.001336393179371953, 0.0003267655265517533, 8.741358215047512e-07, 1.7415688489563763e-05, 2.783414174700738e-07, 0.00029665243346244097, 0.9929754734039307, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.12262558937072754, 0.02392657659947872, 0.03827648609876633, 0.016171958297491074, 0.030366230756044388, 0.29375940561294556, 0.10661352425813675, 0.0705994963645935, 0.04190007597208023, 0.017833102494478226, 0.23792758584022522, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0008121057762764394, 0.00011546660243766382, 0.00013500906061381102, 5.5392923968611285e-05, 0.0001317415590165183, 1.0027196140072192e-06, 1.3799102589473478e-06, 1.6001629887796298e-07, 1.8972989437315846e-06, 5.419660737970844e-07, 4.753100597554294e-07, 0.9987448453903198, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0013333020033314824, 0.00010533027671044692, 0.0007669601473025978, 0.0009021890582516789, 3.5708292216440896e-06, 8.120475740724942e-07, 1.4038076301403635e-07, 9.5612755046659e-08, 1.6865038787727826e-06, 2.9789011023240164e-05, 4.3813363959088747e-07, 1.1965776138822548e-06, 0.9968544840812683, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08159798383712769, 0.021388858556747437, 0.028877155855298042, 0.012675793841481209, 0.02493158169090748, 0.229964941740036, 0.10401230305433273, 0.06472097337245941, 0.03460092470049858, 0.015198714099824429, 0.18564164638519287, 0.020528046414256096, 0.005003852304071188, 0.17085722088813782, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0013488862896338105, 8.92121170181781e-05, 0.000685996375977993, 0.0006963219493627548, 2.1248622942948714e-05, 3.7633941474268795e-07, 6.2267383782455e-07, 4.3072165567537013e-07, 7.091880888765445e-06, 0.0013103694655001163, 1.611375211041377e-07, 9.54876213654643e-07, 0.00016453623538836837, 1.15319252813606e-07, 0.9956737160682678, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0010867505334317684, 0.000407249026466161, 0.00010700321581680328, 0.000678271462675184, 0.00011028484004782513, 1.318426257057581e-07, 2.660271150034532e-07, 4.6195768277357274e-07, 8.409714610024821e-06, 0.0007784898625686765, 5.109449574547398e-08, 8.792079029262823e-07, 4.934294702252373e-05, 3.4893233191723994e-08, 0.000114411988761276, 0.9966580867767334, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05868088826537132, 0.0187857486307621, 0.024067312479019165, 0.011013144627213478, 0.021456381306052208, 0.1911056488752365, 0.09902721643447876, 0.06065867841243744, 0.03016272373497486, 0.014801910147070885, 0.15370330214500427, 0.016615159809589386, 0.004385382868349552, 0.14085331559181213, 0.015014998614788055, 0.0053902058862149715, 0.13427793979644775, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.07323037832975388, 0.02837456949055195, 0.02105824276804924, 0.007611881475895643, 0.01005358062684536, 0.06553948670625687, 0.14559759199619293, 0.08620844036340714, 0.01024126447737217, 0.007425738964229822, 0.045262690633535385, 0.006621456239372492, 0.001162177650257945, 0.03923853859305382, 0.008256880566477776, 0.0030656892340630293, 0.036122631281614304, 0.40492871403694153, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [7.94449370005168e-05, 1.8382123698756914e-06, 3.7134324060161816e-08, 1.5575307770632207e-05, 3.4723757380561437e-06, 1.3165096790501707e-09, 2.703870904952055e-08, 1.7186652101486288e-08, 9.2322629541286e-08, 1.350637921859743e-05, 4.4787887287789374e-10, 2.041673052843862e-09, 5.5420287026208825e-08, 3.0638666603799436e-10, 4.217719435928302e-07, 7.065880708978511e-06, 2.424439260462208e-10, 1.4397358860662735e-09, 0.9998784065246582, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.002934493590146303, 0.0005883841076865792, 0.004250307101756334, 0.0040284618735313416, 0.00011592944065341726, 4.3316217670508195e-06, 6.773899713152787e-06, 7.076467591105029e-05, 0.0002495265216566622, 0.0003159338375553489, 2.08243977795064e-06, 4.4092124880990013e-05, 0.00030446931486949325, 1.4702561657031765e-06, 0.0001976135972654447, 0.0013599261874333024, 1.2478651569836074e-06, 6.875188319099834e-06, 0.00019261146371718496, 0.9853246808052063, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04056532680988312, 0.017898326739668846, 0.020600972697138786, 0.009821100160479546, 0.01788281463086605, 0.15653997659683228, 0.09208228439092636, 0.05628826469182968, 0.026661338284611702, 0.014471272937953472, 0.12537629902362823, 0.013986770063638687, 0.00392792746424675, 0.11453618854284286, 0.013395379297435284, 0.004846664611250162, 0.10880246758460999, 0.049533966928720474, 0.005186409689486027, 0.004420871380716562, 0.10317535698413849, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.02453630231320858, 0.0031243355479091406, 0.0009699289221316576, 4.6315435611177236e-05, 0.0001343475450994447, 0.0006453939131461084, 0.011668838560581207, 0.00032713444670662284, 0.0004573040350805968, 2.3976079319254495e-05, 0.0003445417678449303, 4.996419738745317e-05, 2.5757384719327092e-05, 0.0002821775560732931, 6.525318895000964e-06, 4.981805432180408e-06, 0.00023554006475023925, 0.0002664509229362011, 2.531113386794459e-05, 2.9761044061160646e-05, 0.00018809252651408315, 0.9566071033477783, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.002076001837849617, 0.00014591813669539988, 5.28989803569857e-05, 4.218077810946852e-05, 4.7387235099449754e-05, 4.14233511492057e-07, 0.00012964299821760505, 2.4251021386589855e-05, 1.8034043023362756e-06, 1.4839295090496307e-06, 1.4490007060885546e-07, 4.199936029181117e-06, 4.1068410894240515e-08, 1.0260500005188078e-07, 8.285463763968437e-07, 5.781907930213492e-07, 7.69515864362802e-08, 6.669622365507166e-08, 6.47262713755481e-05, 1.5700916264904663e-06, 5.6418304694716426e-08, 4.403035745781381e-06, 0.997401237487793, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0005159107386134565, 0.00011814631579909474, 3.2705611374694854e-05, 0.00021188720711506903, 4.563520269584842e-05, 2.924884086041857e-07, 3.0579394660890102e-06, 7.607361141026558e-08, 0.0001375367137370631, 0.0005283543141558766, 1.1576013747571778e-07, 4.028577222925378e-06, 2.079294745271909e-06, 8.805408668877135e-08, 0.00010051687422674149, 4.43161916336976e-05, 6.963698950812613e-08, 9.373040654736542e-08, 3.337768794153817e-05, 1.5888888810877688e-05, 5.6106753021367695e-08, 7.05264184830412e-08, 9.389270417159423e-06, 0.9981963038444519, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.06915225088596344, 0.014032105915248394, 0.005599097348749638, 0.0012576853623613715, 0.0008468001033179462, 0.009601173922419548, 0.008965768851339817, 0.052476901561021805, 0.00025028391974046826, 0.0009063858306035399, 0.005419905763119459, 0.000730921165086329, 0.0001918794005177915, 0.004396580625325441, 0.005964720156043768, 0.00035585410660132766, 0.0038770169485360384, 0.009727226570248604, 0.0011023489059880376, 0.0003644288226496428, 0.0034139587078243494, 0.0030523554887622595, 0.005698245484381914, 0.00029557300149463117, 0.7923205494880676, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0010461547644808888, 1.9557141058612615e-05, 1.7358959212288028e-06, 3.1030376703711227e-05, 2.73079531325493e-05, 1.0180698062356441e-08, 4.815767624677392e-07, 1.749424853869641e-07, 7.31240987761339e-08, 7.983068712746899e-07, 3.4643101720632785e-09, 4.9067175496020354e-06, 1.0718932230702194e-07, 2.352543715389288e-09, 2.3834076046114205e-07, 1.3731333865507622e-06, 1.828867168818249e-09, 6.247609984910696e-09, 9.208339179167524e-06, 1.6384256014134735e-06, 1.2523899695082719e-09, 8.271274221094416e-10, 8.456770046905149e-06, 3.118676431768108e-06, 9.241420961814129e-09, 0.9988435506820679, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0010544945253059268, 0.00021373217168729752, 0.00028157219639979303, 0.0011373378802090883, 0.00014783968799747527, 2.00685326490202e-06, 4.831355909118429e-06, 3.48233857039304e-06, 5.100344878883334e-06, 0.000124204860185273, 9.301529075855797e-07, 1.0582622962829191e-05, 1.1688286576827522e-05, 7.077518944242911e-07, 0.003411600599065423, 0.00019905276712961495, 5.995618153065152e-07, 4.643046622732072e-07, 1.9585138943511993e-05, 0.00011087791790487245, 4.7500583377768635e-07, 3.2808881655910227e-07, 4.535358584689675e-06, 5.1617142162285745e-05, 3.247476740853017e-07, 1.3071385183138773e-05, 0.9931889772415161, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.02420075610280037, 0.002820128807798028, 0.004319296684116125, 0.0009108306840062141, 0.0017753936117514968, 0.003809914458543062, 0.0029533228371292353, 0.01671477034687996, 0.001393173704855144, 0.005003183614462614, 0.002276088809594512, 0.0005876408540643752, 0.0009901623707264662, 0.0018066932680085301, 0.0031312420032918453, 0.00023389369016513228, 0.0016082713846117258, 0.00991030503064394, 0.002853952581062913, 0.00020495128410402685, 0.0013712650397792459, 0.00691900635138154, 0.0010336573468521237, 0.00014528384781442583, 0.017489798367023468, 0.00046553125139325857, 0.00030240853084251285, 0.8847692012786865, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0005256092990748584, 3.528650950102019e-06, 1.703768475636025e-06, 9.408179903402925e-05, 0.6647976040840149, 1.6531491553450905e-07, 9.511112466498162e-07, 8.886883620107255e-07, 1.701671317277942e-05, 3.7421868910314515e-05, 6.832266308265389e-08, 9.810898518480826e-06, 7.548737102069936e-08, 4.9632518539510784e-08, 2.0393888462422183e-06, 7.686546268814709e-06, 4.066851388984105e-08, 2.6744315206883584e-08, 3.343187927384861e-05, 4.341078749803273e-07, 3.007988169656528e-08, 3.206056931048806e-08, 4.900574822386261e-06, 3.0609394343628082e-06, 1.4603195985785078e-08, 2.2145539332996123e-05, 9.21115315577481e-06, 3.164272044386962e-08, 0.33442792296409607, 0.0, 0.0, 0.0, 0.0], [0.004677004646509886, 0.0002802134840749204, 8.346093818545341e-05, 3.157015271426644e-06, 4.065350367454812e-07, 1.334351367177078e-07, 2.6423560484545305e-05, 1.14399881567806e-07, 2.787234905099467e-07, 3.7496099594136467e-06, 4.652897445112103e-08, 1.7213982346220291e-06, 1.8165600579322927e-07, 3.1484699292150253e-08, 1.8334249318741058e-07, 5.2147463236451586e-08, 2.5215808108214333e-08, 3.078440968806717e-08, 7.142029545548212e-08, 1.6640171907056356e-08, 1.7768222448921733e-08, 8.634525272555038e-08, 2.80096719507128e-06, 8.454135809188301e-07, 3.098357126418705e-08, 2.927866091795295e-07, 5.67622805647261e-07, 5.0554116448608966e-09, 2.9141034474378102e-08, 0.9949179887771606, 0.0, 0.0, 0.0], [5.636792411678471e-05, 0.00020023711840622127, 1.6319079350068932e-06, 1.5089327689565835e-06, 1.7779733752831817e-05, 5.95644145118257e-10, 6.940369075891795e-06, 5.034019867622419e-09, 2.1682852491267113e-07, 1.7164256860269234e-06, 1.9326265587871205e-10, 1.8911425314627195e-08, 9.538840117784275e-08, 1.2946883842790413e-10, 8.07715050399338e-09, 4.386615546536632e-05, 9.768683778554887e-11, 5.256080015669795e-10, 6.93216934450902e-05, 7.505880716962565e-08, 7.069290985928234e-11, 3.988046870517792e-08, 1.0610276149236597e-05, 2.2120707399153616e-06, 8.688633701403603e-10, 9.928487088473048e-06, 5.390084378831261e-09, 1.1354458345769203e-10, 3.060037442992325e-06, 3.8945552205404965e-07, 0.9995738863945007, 0.0, 0.0], [0.00035569758620113134, 1.9489458281896077e-05, 9.160975423583295e-06, 1.3085252248856705e-05, 2.3808875994291157e-05, 2.97190183573548e-08, 4.1605944716138765e-05, 1.7756508441379992e-07, 6.896241870890663e-07, 1.050624064191652e-06, 1.2230765733534099e-08, 4.290548076824052e-06, 3.295830310889869e-07, 9.21108966878137e-09, 1.7466413737565745e-06, 4.68278949483647e-06, 7.4427397578347154e-09, 9.673819079125678e-09, 3.9665097574470565e-05, 7.324898660954204e-07, 5.5436903956262995e-09, 1.0396706073834139e-07, 1.4401575754163787e-05, 8.873340448189992e-06, 5.98941607421466e-09, 1.5627783795935102e-05, 2.9811219519615406e-06, 6.874342872720263e-09, 4.854650796914939e-06, 7.0722649070376065e-06, 6.210685569385532e-06, 0.999423623085022, 0.0], [0.05455271154642105, 0.01642906479537487, 0.01600833795964718, 0.006986881140619516, 0.02488415502011776, 0.07376932352781296, 0.07933780550956726, 0.05594552680850029, 0.02726820856332779, 0.013368779793381691, 0.05590308830142021, 0.007223027292639017, 0.005426580552011728, 0.05012328550219536, 0.00827906746417284, 0.0030609637033194304, 0.046822965145111084, 0.04751850664615631, 0.0028449445962905884, 0.007037974428385496, 0.042827170342206955, 0.0513819120824337, 0.02728237397968769, 0.026375338435173035, 0.026159211993217468, 0.011997989378869534, 0.006220121867954731, 0.03648780658841133, 0.011124704033136368, 0.005218563135713339, 0.0025592022575438023, 0.0026040268130600452, 0.14697040617465973]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9192656874656677, 0.08073433488607407, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.45437392592430115, 0.4265042543411255, 0.11912186443805695, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5267521739006042, 0.22321631014347076, 0.12420626729726791, 0.12582528591156006, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3903489112854004, 0.17099407315254211, 0.06872710585594177, 0.1837182343006134, 0.1862117052078247, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.19744226336479187, 0.25436851382255554, 0.14488449692726135, 0.18925853073596954, 0.20821937918663025, 0.00582678010687232, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.19473573565483093, 0.13241828978061676, 0.14391927421092987, 0.15120737254619598, 0.3002447485923767, 0.02058463543653488, 0.056889984756708145, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1269666850566864, 0.21597377955913544, 0.16089318692684174, 0.1969882994890213, 0.1884462684392929, 0.013544929213821888, 0.07520709931850433, 0.021979698911309242, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2912123203277588, 0.05954226851463318, 0.11079408973455429, 0.09717857837677002, 0.22345606982707977, 0.016262585297226906, 0.08719441294670105, 0.024762287735939026, 0.08959739655256271, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.22981953620910645, 0.038026757538318634, 0.1601383537054062, 0.11780407279729843, 0.1618119478225708, 0.03552818298339844, 0.07561112195253372, 0.020756039768457413, 0.10670627653598785, 0.05379769951105118, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.13314427435398102, 0.18858656287193298, 0.10808095335960388, 0.15994971990585327, 0.1773921251296997, 0.003598014824092388, 0.0250620748847723, 0.01286292728036642, 0.0745309367775917, 0.11275240778923035, 0.004039982799440622, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1674640029668808, 0.1435522586107254, 0.06118571758270264, 0.17358075082302094, 0.10113830864429474, 0.01755347289144993, 0.03239530324935913, 0.007995500229299068, 0.05453447997570038, 0.07883201539516449, 0.017258215695619583, 0.14450997114181519, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.26996830105781555, 0.13029062747955322, 0.06605685502290726, 0.061561424285173416, 0.08437130600214005, 0.01984662190079689, 0.01631278730928898, 0.012965923175215721, 0.03231464698910713, 0.05254792049527168, 0.01986861415207386, 0.13695642352104187, 0.09693864732980728, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09820353239774704, 0.13618646562099457, 0.0800577849149704, 0.12152864784002304, 0.1357937604188919, 0.0024323570542037487, 0.01778949797153473, 0.009086488746106625, 0.05637527257204056, 0.08521638810634613, 0.0027533371467143297, 0.1520378440618515, 0.0994671955704689, 0.00307140382938087, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1701289564371109, 0.08885712921619415, 0.09089689701795578, 0.08085790276527405, 0.11427336931228638, 0.010901479050517082, 0.014788412488996983, 0.005497370846569538, 0.051512569189071655, 0.05330684036016464, 0.010595815256237984, 0.09442280977964401, 0.13098116219043732, 0.011153682135045528, 0.07182558625936508, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10365856438875198, 0.03588540852069855, 0.08803567290306091, 0.06087937951087952, 0.08348330855369568, 0.03080574795603752, 0.029615936800837517, 0.010918705724179745, 0.04049992188811302, 0.04288605973124504, 0.03219296410679817, 0.15008129179477692, 0.12658776342868805, 0.0349021814763546, 0.05825432389974594, 0.07131282985210419, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.07047097384929657, 0.0960099846124649, 0.056215789169073105, 0.08699887990951538, 0.09934122860431671, 0.0016298787668347359, 0.01240861602127552, 0.006380431819707155, 0.041014038026332855, 0.06348712742328644, 0.0018482195446267724, 0.11387647688388824, 0.07328979671001434, 0.0020712940022349358, 0.07246166467666626, 0.20012779533863068, 0.002367778215557337, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05659440532326698, 0.08906208723783493, 0.04703241214156151, 0.09457876533269882, 0.08793889731168747, 0.001972166122868657, 0.017767680808901787, 0.005990432109683752, 0.06014860421419144, 0.08834964036941528, 0.0022637685760855675, 0.1046152412891388, 0.06079366058111191, 0.00255662202835083, 0.07818624377250671, 0.19496025145053864, 0.0029299911111593246, 0.004259058274328709, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1597062051296234, 0.026096442714333534, 0.04515180364251137, 0.0349758081138134, 0.08030673116445541, 0.05469036474823952, 0.04078545421361923, 0.02651304565370083, 0.02725193277001381, 0.03919816389679909, 0.05351434648036957, 0.0484292209148407, 0.0626426488161087, 0.05558208376169205, 0.05559664964675903, 0.04792959615588188, 0.05676326900720596, 0.04892301186919212, 0.035943206399679184, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08663122355937958, 0.09557436406612396, 0.036738116294145584, 0.06946705281734467, 0.05681459605693817, 0.00819021463394165, 0.022195832803845406, 0.0076428246684372425, 0.03146827220916748, 0.07555203884840012, 0.0089652631431818, 0.05263480916619301, 0.04491106793284416, 0.009935053065419197, 0.04481377825140953, 0.15198466181755066, 0.0109334047883749, 0.007993670180439949, 0.16402564942836761, 0.01352810300886631, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04783454164862633, 0.06669356673955917, 0.03798917308449745, 0.06040368974208832, 0.06899043172597885, 0.0010267288889735937, 0.008172674104571342, 0.004158170893788338, 0.028268268331885338, 0.044072382152080536, 0.0011586467735469341, 0.0783291831612587, 0.05082900822162628, 0.0013004952343180776, 0.049042947590351105, 0.14145004749298096, 0.0014934344217181206, 0.0025566834956407547, 0.17534081637859344, 0.12910208106040955, 0.0017870558658614755, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04114089906215668, 0.04766072705388069, 0.03586998209357262, 0.06666103005409241, 0.07969274371862411, 0.00236507598310709, 0.02034018747508526, 0.004854495171457529, 0.0643620491027832, 0.06737152487039566, 0.0027073686942458153, 0.05596053972840309, 0.05648668855428696, 0.0030767249409109354, 0.05354386568069458, 0.15165923535823822, 0.003539716824889183, 0.0046739280223846436, 0.16402718424797058, 0.05968156456947327, 0.004154939670115709, 0.010169521905481815, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0662880688905716, 0.04092540964484215, 0.02732268162071705, 0.04505367949604988, 0.06575217843055725, 0.015657562762498856, 0.02788187935948372, 0.013711260631680489, 0.03174294903874397, 0.05280746519565582, 0.015447776764631271, 0.032181113958358765, 0.032901059836149216, 0.01629886031150818, 0.0266366358846426, 0.09772849082946777, 0.017114203423261642, 0.01691945269703865, 0.09847065806388855, 0.04787908494472504, 0.018412545323371887, 0.015331599861383438, 0.17753532528877258, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09801430255174637, 0.026328807696700096, 0.03586815297603607, 0.05713058263063431, 0.06943874061107635, 0.012128958478569984, 0.020909087732434273, 0.008747960440814495, 0.02699965611100197, 0.037266768515110016, 0.011716341599822044, 0.08499539643526077, 0.05501333996653557, 0.012196713127195835, 0.03563909977674484, 0.08789661526679993, 0.012970106676220894, 0.010851015336811543, 0.06408385187387466, 0.059639204293489456, 0.01449217926710844, 0.006093734409660101, 0.09761437773704529, 0.053964968770742416, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.02769325114786625, 0.04073771461844444, 0.031223542988300323, 0.058790355920791626, 0.056018248200416565, 0.0015714796027168632, 0.011908041313290596, 0.003339792136102915, 0.051765501499176025, 0.05291244387626648, 0.0018259818898513913, 0.060300350189208984, 0.028357185423374176, 0.0020878082141280174, 0.05086478963494301, 0.07898838073015213, 0.0023987055756151676, 0.002904822351410985, 0.11135734617710114, 0.04586018621921539, 0.0028849171940237284, 0.011431250721216202, 0.1234055683016777, 0.13488474488258362, 0.006487556733191013, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15390776097774506, 0.023786796256899834, 0.05320245772600174, 0.03963885083794594, 0.05711062252521515, 0.015628937631845474, 0.02020607329905033, 0.009252368472516537, 0.02131185494363308, 0.02902994491159916, 0.014741902239620686, 0.0612383633852005, 0.10112591087818146, 0.015157817862927914, 0.04335402697324753, 0.04090719670057297, 0.01573265716433525, 0.020073410123586655, 0.053749360144138336, 0.03468616306781769, 0.01698937825858593, 0.006368495989590883, 0.04042017087340355, 0.031060419976711273, 0.013937168754637241, 0.06738192588090897, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08555091917514801, 0.035057418048381805, 0.05885257571935654, 0.03497908264398575, 0.05178743600845337, 0.0025119921192526817, 0.0041512190364301205, 0.0020859173964709044, 0.014081120491027832, 0.02449135296046734, 0.0023760052863508463, 0.09535116702318192, 0.07766135036945343, 0.0024908811319619417, 0.045726776123046875, 0.05774116516113281, 0.002660380443558097, 0.0033477561082690954, 0.047998279333114624, 0.02678881771862507, 0.0029587154276669025, 0.004655787255614996, 0.03607099875807762, 0.021599192172288895, 0.004788633435964584, 0.22052206099033356, 0.03371305391192436, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.020706716924905777, 0.037194062024354935, 0.01972370594739914, 0.038942236453294754, 0.03182699903845787, 0.0012253678869456053, 0.008815747685730457, 0.002405293518677354, 0.0345795564353466, 0.047298070043325424, 0.001395374652929604, 0.05148206278681755, 0.02048674412071705, 0.0015608376124873757, 0.03826117143034935, 0.04863753169775009, 0.0017825308023020625, 0.0025446040090173483, 0.06541350483894348, 0.0418856181204319, 0.0020960604306310415, 0.009401390329003334, 0.08460027724504471, 0.09019991755485535, 0.0054032751359045506, 0.22932568192481995, 0.05692270025610924, 0.005882916506379843, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08582809567451477, 0.039671752601861954, 0.014841826632618904, 0.056536220014095306, 0.05960613489151001, 0.0069875167682766914, 0.010110098868608475, 0.002887725830078125, 0.015322853811085224, 0.02592875435948372, 0.006830447353422642, 0.033548954874277115, 0.021093523129820824, 0.007213158532977104, 0.019323380663990974, 0.05298228561878204, 0.007658614311367273, 0.005157842300832272, 0.06523183733224869, 0.018170006573200226, 0.008534705266356468, 0.0043681650422513485, 0.026912428438663483, 0.049627941101789474, 0.004404331557452679, 0.19388671219348907, 0.03933224827051163, 0.005363550037145615, 0.11263895034790039, 0.0, 0.0, 0.0, 0.0], [0.07040347903966904, 0.05926162749528885, 0.015334847383201122, 0.0241390373557806, 0.015728840604424477, 0.019492102786898613, 0.017311513423919678, 0.017303066328167915, 0.020563244819641113, 0.0451558418571949, 0.021815825253725052, 0.07079727947711945, 0.030063964426517487, 0.02354060858488083, 0.020684480667114258, 0.02116367779672146, 0.02644512802362442, 0.023540044203400612, 0.05663726478815079, 0.0336947925388813, 0.030707288533449173, 0.015993360430002213, 0.015474079176783562, 0.030803624540567398, 0.03180558606982231, 0.07969609647989273, 0.03437487408518791, 0.034537144005298615, 0.030624371021986008, 0.06290692836046219, 0.0, 0.0, 0.0], [0.05338500067591667, 0.029621528461575508, 0.021030094474554062, 0.023695707321166992, 0.033436402678489685, 0.01839382015168667, 0.024060707539319992, 0.02281002327799797, 0.013972364366054535, 0.028813544660806656, 0.019059743732213974, 0.015371965244412422, 0.02533530630171299, 0.019971953704953194, 0.014075737446546555, 0.0388445146381855, 0.021346209570765495, 0.019321434199810028, 0.056326594203710556, 0.021360008046030998, 0.023394575342535973, 0.023136639967560768, 0.06416381895542145, 0.06724688410758972, 0.0346950888633728, 0.043490175157785416, 0.03032725676894188, 0.03285712003707886, 0.053275078535079956, 0.0659981295466423, 0.041182488203048706, 0.0, 0.0], [0.18177089095115662, 0.019533155485987663, 0.028002941980957985, 0.02911018393933773, 0.04134833440184593, 0.01801948994398117, 0.010471724905073643, 0.008382909931242466, 0.01533578522503376, 0.021839451044797897, 0.01689334586262703, 0.03806138411164284, 0.03643865883350372, 0.017464255914092064, 0.026129068806767464, 0.04011504724621773, 0.018317215144634247, 0.012488218024373055, 0.05136970803141594, 0.01944384165108204, 0.019889770075678825, 0.0035764521453529596, 0.026232963427901268, 0.018429189920425415, 0.011723755858838558, 0.0327298603951931, 0.034808199852705, 0.01203504391014576, 0.06030292063951492, 0.00787330698221922, 0.042272936552762985, 0.07959000021219254, 0.0], [0.015003404580056667, 0.02033485472202301, 0.007813435047864914, 0.016553575173020363, 0.01646609604358673, 0.00023809655976947397, 0.0019598929211497307, 0.0006898260326124728, 0.006483218166977167, 0.010734074749052525, 0.0002443529956508428, 0.014634673483669758, 0.00924762524664402, 0.0002605919726192951, 0.009339917451143265, 0.042726580053567886, 0.0002886394504457712, 0.0003811670758295804, 0.054749373346567154, 0.09452768415212631, 0.00034211305319331586, 0.001727019902318716, 0.04745933413505554, 0.02661379985511303, 0.0010238662362098694, 0.1653798222541809, 0.014187106862664223, 0.0013092352310195565, 0.04770654812455177, 0.0111984983086586, 0.2205076366662979, 0.1394261121749878, 0.0004418626776896417]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9333657622337341, 0.06663428246974945, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.32452359795570374, 0.5923718214035034, 0.08310455828905106, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1415974646806717, 0.514380931854248, 0.28144142031669617, 0.06258013844490051, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.25262272357940674, 0.058340705931186676, 0.10458870232105255, 0.4411159157752991, 0.14333190023899078, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.18051563203334808, 0.09828907996416092, 0.10873956233263016, 0.12262694537639618, 0.09995546191930771, 0.38987332582473755, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1719229817390442, 0.023320995271205902, 0.12332706153392792, 0.06076495721936226, 0.05971711874008179, 0.347591757774353, 0.21335510909557343, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09627032279968262, 0.04683619737625122, 0.028257304802536964, 0.03239249438047409, 0.0586504191160202, 0.15829356014728546, 0.3099648952484131, 0.2693347930908203, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09948235750198364, 0.0646418035030365, 0.028671829029917717, 0.0730566680431366, 0.02978716976940632, 0.1396002322435379, 0.20754705369472504, 0.265802264213562, 0.09141054004430771, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05632037669420242, 0.029885854572057724, 0.01341991126537323, 0.0024075533729046583, 0.013245469890534878, 0.045645419508218765, 0.05522569268941879, 0.10027101635932922, 0.6579986810684204, 0.02558007836341858, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04266994819045067, 0.010426479391753674, 0.010652213357388973, 0.013446621596813202, 0.00935314130038023, 0.04015978425741196, 0.0710517093539238, 0.1373765617609024, 0.10423263162374496, 0.21736638247966766, 0.34326446056365967, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0296839140355587, 0.002561642089858651, 0.004407483618706465, 0.034248705953359604, 0.014238743111491203, 0.028001822531223297, 0.02228802628815174, 0.07105964422225952, 0.04026233032345772, 0.5543041825294495, 0.17333054542541504, 0.025612974539399147, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04039592295885086, 0.008631582371890545, 0.005494222976267338, 0.003864138852804899, 0.023363104090094566, 0.027207026258111, 0.025677336379885674, 0.050485219806432724, 0.0510561428964138, 0.05041782185435295, 0.14164337515830994, 0.511528491973877, 0.060235634446144104, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.028706390410661697, 0.004808460362255573, 0.004723656922578812, 0.005382601171731949, 0.003456039587035775, 0.014470163732767105, 0.024037377908825874, 0.041626207530498505, 0.03567296266555786, 0.07060738652944565, 0.10830992460250854, 0.05811614543199539, 0.2062121331691742, 0.3938705325126648, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05342644825577736, 0.011923279613256454, 0.015572259202599525, 0.0021123222541064024, 0.005851038731634617, 0.02217092178761959, 0.03818469122052193, 0.02741156704723835, 0.054642871022224426, 0.04910830780863762, 0.12328378856182098, 0.05287173017859459, 0.07545649260282516, 0.3701530694961548, 0.09783118963241577, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.008346711285412312, 0.00027852741186507046, 0.0011785993119701743, 0.0021395485382527113, 0.002476169029250741, 0.002918292535468936, 0.004576893523335457, 0.0039655971340835094, 0.004973022732883692, 0.011347771622240543, 0.013014075346291065, 0.005545974709093571, 0.02089560590684414, 0.03838539123535156, 0.8750316500663757, 0.004926194902509451, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.02170032449066639, 0.002678664866834879, 0.0023839243222028017, 0.002815627260133624, 0.001556233735755086, 0.006139412056654692, 0.009039806202054024, 0.01374087855219841, 0.012412214651703835, 0.02468654327094555, 0.03406905382871628, 0.017561480402946472, 0.06082957610487938, 0.12004052847623825, 0.11516803503036499, 0.12663689255714417, 0.42854079604148865, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.01968139037489891, 0.0028661887627094984, 0.002752861240878701, 0.0023642012383788824, 0.0022205759305506945, 0.0047769746743142605, 0.005015391390770674, 0.007020775228738785, 0.006603003013879061, 0.01582488976418972, 0.02177913300693035, 0.02237590402364731, 0.03905554488301277, 0.07342638820409775, 0.07227564603090286, 0.0810362920165062, 0.2556319534778595, 0.36529290676116943, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.034438684582710266, 0.0024295076727867126, 0.0027038780972361565, 0.0010108931455761194, 0.0028212429024279118, 0.009310661815106869, 0.00747407041490078, 0.010526392608880997, 0.01934259757399559, 0.006583388429135084, 0.030525274574756622, 0.008166542276740074, 0.023945646360516548, 0.08047989010810852, 0.04428938403725624, 0.048865143209695816, 0.23266319930553436, 0.38709720969200134, 0.04732644557952881, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0036976831033825874, 0.006450952962040901, 0.0004921727231703699, 0.0003069589729420841, 0.00012400773994158953, 0.0010727370390668511, 0.0008858796209096909, 0.001380636473186314, 0.0008200175943784416, 0.0002731183485593647, 0.002783065428957343, 0.03529426082968712, 0.0035613791551440954, 0.007148314733058214, 0.001647408353164792, 0.016731660813093185, 0.01917281188070774, 0.022339196875691414, 0.8733668327331543, 0.002450960921123624, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.013628787361085415, 0.001438445644453168, 0.0012188641121611, 0.0012915851548314095, 0.0006118023884482682, 0.0021400381810963154, 0.0027469571214169264, 0.0033360477536916733, 0.003091872902587056, 0.005988912656903267, 0.007237219717353582, 0.003547744592651725, 0.012891951948404312, 0.022229159250855446, 0.020824221894145012, 0.02629532851278782, 0.0764816403388977, 0.16261443495750427, 0.1263866275548935, 0.10034442692995071, 0.4056538939476013, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.021432969719171524, 0.0007291649235412478, 0.0010889691766351461, 0.0008559620473533869, 0.0010556603083387017, 0.002354251453652978, 0.0018633343279361725, 0.002486992860212922, 0.004011375363916159, 0.0036456233356148005, 0.006636620499193668, 0.0024591651745140553, 0.00810664240270853, 0.01950138807296753, 0.013945515267550945, 0.01683160476386547, 0.06727856397628784, 0.1020374447107315, 0.07506179064512253, 0.03785506263375282, 0.3474159240722656, 0.2633460462093353, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.01837441883981228, 0.0016340153524652123, 0.0002648660447448492, 0.0006277099018916488, 0.0007212890777736902, 0.0022947092074900866, 0.0014375777682289481, 0.0024335708003491163, 0.0010225051082670689, 0.002173094777390361, 0.005665970034897327, 0.000585706380661577, 0.004730311222374439, 0.01578962802886963, 0.009121065959334373, 0.006447889842092991, 0.049651872366666794, 0.07221098989248276, 0.04530426487326622, 0.02253621071577072, 0.23380489647388458, 0.28100958466529846, 0.22215785086154938, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.01601027511060238, 0.00041907597915269434, 0.0016793111572042108, 0.0020331626292318106, 0.0011126026511192322, 0.0023041211534291506, 0.001641488284803927, 0.0020452297758311033, 0.0008826447883620858, 0.003531813621520996, 0.00466074887663126, 0.001146126538515091, 0.004025280009955168, 0.011729470454156399, 0.006906012073159218, 0.005978981498628855, 0.03413612022995949, 0.053919851779937744, 0.07536919414997101, 0.015544144436717033, 0.153072789311409, 0.21672524511814117, 0.23267339169979095, 0.1524529755115509, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.01641550101339817, 0.0010881004855036736, 0.001302703283727169, 0.001233565155416727, 0.0010325239272788167, 0.0013605316635221243, 0.001135275000706315, 0.0009713207837194204, 0.0017580230487510562, 0.002354995347559452, 0.0022677984088659286, 0.0026170071214437485, 0.0032849612180143595, 0.00556444376707077, 0.0072877430357038975, 0.006582668051123619, 0.0169111005961895, 0.023348137736320496, 0.033363793045282364, 0.01206086203455925, 0.08410514146089554, 0.10776982456445694, 0.12741318345069885, 0.2500958740711212, 0.2886749505996704, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.031117741018533707, 0.0015063054161146283, 0.0010839324677363038, 0.002421524142846465, 0.001382686896249652, 0.0037937266752123833, 0.001633929554373026, 0.003119518980383873, 0.0012812635395675898, 0.002068523084744811, 0.0047601195983588696, 0.0008757320465520024, 0.008592123165726662, 0.0093536376953125, 0.011993243359029293, 0.005963001865893602, 0.022028174251317978, 0.03411327302455902, 0.009586759842932224, 0.013681814074516296, 0.08417054265737534, 0.08917812258005142, 0.10806837677955627, 0.15560784935951233, 0.37236863374710083, 0.020249491557478905, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.014838481321930885, 0.0005757318576797843, 0.004743653815239668, 0.00045677347225137055, 0.0007412288105115294, 0.0015000010607764125, 0.0008866073912940919, 0.0008452099282294512, 0.00010359263978898525, 0.000906514236703515, 0.0018096797866746783, 0.0602911040186882, 0.001537756179459393, 0.0033079395070672035, 0.0013713664375245571, 0.002082675462588668, 0.008643986657261848, 0.009088865481317043, 0.007760955020785332, 0.00580833712592721, 0.033865127712488174, 0.03987184166908264, 0.05374552309513092, 0.036947719752788544, 0.12259436398744583, 0.5718704462051392, 0.013804465532302856, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.01857001520693302, 0.0012143177445977926, 0.001288044499233365, 0.0010612568585202098, 0.0007490046555176377, 0.001048562815412879, 0.000868787697982043, 0.0005973864463157952, 0.0012938270810991526, 0.001404251204803586, 0.0011055096983909607, 0.0005003446131013334, 0.0020262624602764845, 0.0021463141310960054, 0.0028737925458699465, 0.0026413712184876204, 0.005802977364510298, 0.0081913648173213, 0.00833908375352621, 0.005543305538594723, 0.026558682322502136, 0.03308920934796333, 0.032507359981536865, 0.09199848771095276, 0.10410483181476593, 0.16861778497695923, 0.208954319357872, 0.2669035494327545, 0.0, 0.0, 0.0, 0.0, 0.0], [0.016654442995786667, 0.0007248665788210928, 0.0008980666170828044, 0.0031616878695786, 0.0005522824940271676, 0.0016607288271188736, 0.00074861227767542, 0.0009652797016315162, 0.0003842804580926895, 0.0022577813360840082, 0.0015545616624876857, 0.0007037006434984505, 0.0014688181690871716, 0.002799882087856531, 0.003013978712260723, 0.005369038321077824, 0.006552346982061863, 0.008701568469405174, 0.007138868793845177, 0.006516328547149897, 0.026185575872659683, 0.03929203003644943, 0.03155006840825081, 0.02434305101633072, 0.1067076250910759, 0.15159794688224792, 0.13674212992191315, 0.3201395273208618, 0.09161490201950073, 0.0, 0.0, 0.0, 0.0], [0.0372268483042717, 0.0007442747591994703, 0.0019180340459570289, 0.0014290640829131007, 0.0025597368367016315, 0.003292836481705308, 0.0009007275220938027, 0.001544511178508401, 0.0003557214804459363, 0.0004492733278311789, 0.0026688158977776766, 0.0014046661090105772, 0.0017281017499044538, 0.004542736802250147, 0.0012941033346578479, 0.004677668213844299, 0.009138188324868679, 0.008992945775389671, 0.005343662574887276, 0.0020318389870226383, 0.030930904671549797, 0.021215269342064857, 0.03639978542923927, 0.056794628500938416, 0.11195950210094452, 0.011458109132945538, 0.03353285789489746, 0.2658853828907013, 0.2469542920589447, 0.09262555837631226, 0.0, 0.0, 0.0], [0.044088784605264664, 0.0011082949349656701, 0.0033875906374305487, 0.004853191319853067, 0.002864320995286107, 0.004500862676650286, 0.0014518116367980838, 0.0021924942266196012, 0.0007902359939180315, 0.0006717380601912737, 0.003577157622203231, 0.0005131773068569601, 0.012861719354987144, 0.0051168459467589855, 0.0030142527539283037, 0.005887466017156839, 0.010453412309288979, 0.013818331062793732, 0.008691059425473213, 0.011229269206523895, 0.0314009003341198, 0.023181650787591934, 0.03044436313211918, 0.03520682081580162, 0.09999866038560867, 0.006090165581554174, 0.14714214205741882, 0.16393053531646729, 0.12749530375003815, 0.1935775727033615, 0.0004599907260853797, 0.0, 0.0], [0.03717399016022682, 0.005006894003599882, 0.0016112301964312792, 0.0028301470447331667, 0.0005118494154885411, 0.0027071270160377026, 0.0020970311015844345, 0.0012328773736953735, 0.0006310884491540492, 0.001763244392350316, 0.001786878565326333, 0.00025283300783485174, 0.0010720754507929087, 0.0026448338758200407, 0.0015661576762795448, 0.0038084082771092653, 0.005276741925626993, 0.006930254865437746, 0.01331083569675684, 0.0035816438030451536, 0.018304521217942238, 0.026684748008847237, 0.019971538335084915, 0.03215576708316803, 0.07432040572166443, 0.023922255262732506, 0.08761478215456009, 0.1800757199525833, 0.027492573484778404, 0.059722453355789185, 0.19122688472270966, 0.1627122461795807, 0.0], [0.00811693910509348, 0.0008723069913685322, 0.0003707293944898993, 0.00047348643420264125, 0.0001783298939699307, 0.0003994710568804294, 0.00029511720640584826, 0.00018311945314053446, 0.000167145932209678, 0.00023702479666098952, 0.0002533360675442964, 0.00012855060049332678, 0.00020213205425534397, 0.0003994489670731127, 0.0001651181373745203, 0.00046228704741224647, 0.0008780225180089474, 0.0011140474816784263, 0.0020708786323666573, 0.0019636773504316807, 0.0035278573632240295, 0.0044752187095582485, 0.004463486839085817, 0.007312532514333725, 0.014446244575083256, 0.035030465573072433, 0.02001415751874447, 0.04388578608632088, 0.019436758011579514, 0.11325737833976746, 0.07277627289295197, 0.16196827590465546, 0.4804745018482208]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.947374701499939, 0.05262523517012596, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8785883784294128, 0.055394671857357025, 0.06601690500974655, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7270770072937012, 0.1370987594127655, 0.04866093024611473, 0.08716330677270889, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4651746451854706, 0.13429315388202667, 0.1535816341638565, 0.1747254729270935, 0.07222513109445572, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1703597903251648, 0.011841950938105583, 0.0236508846282959, 0.021745815873146057, 0.025856325402855873, 0.7465452551841736, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14214572310447693, 0.04383273795247078, 0.050211820751428604, 0.04516506567597389, 0.058721259236335754, 0.5111533999443054, 0.1487700343132019, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0847334936261177, 0.020369820296764374, 0.03746161609888077, 0.03195912018418312, 0.03419598937034607, 0.43500685691833496, 0.1189405769109726, 0.2373325079679489, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3423079550266266, 0.03593634441494942, 0.0725921243429184, 0.10352761298418045, 0.08196527510881424, 0.07694562524557114, 0.14196324348449707, 0.09896710515022278, 0.045794595032930374, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3711485266685486, 0.06278738379478455, 0.05806950107216835, 0.055662769824266434, 0.13124839961528778, 0.07535629719495773, 0.10726629197597504, 0.07924024015665054, 0.030107343569397926, 0.029113225638866425, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.06430108100175858, 0.0051691061817109585, 0.01101299561560154, 0.009054888971149921, 0.011748063378036022, 0.3188501000404358, 0.03684947267174721, 0.0831955149769783, 0.012068143114447594, 0.014676210470497608, 0.433074414730072, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.23777922987937927, 0.03812997415661812, 0.01974336989223957, 0.07508028298616409, 0.14420795440673828, 0.06244930252432823, 0.07018385827541351, 0.06523600965738297, 0.048832181841135025, 0.03460032865405083, 0.06439767777919769, 0.13935980200767517, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.29014477133750916, 0.04085766524076462, 0.06965630501508713, 0.06467737257480621, 0.11515594273805618, 0.07279439270496368, 0.05406038090586662, 0.040959976613521576, 0.036461226642131805, 0.03709695488214493, 0.08112155646085739, 0.06249447911977768, 0.03451891988515854, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04191606491804123, 0.0033880723640322685, 0.007082367315888405, 0.005899207200855017, 0.007586182560771704, 0.20389790832996368, 0.023296764120459557, 0.05173072963953018, 0.007816805504262447, 0.00973137654364109, 0.2776019871234894, 0.008386771194636822, 0.01617271639406681, 0.3354930877685547, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15129348635673523, 0.04240793362259865, 0.043833814561367035, 0.09706760942935944, 0.14943556487560272, 0.044398821890354156, 0.0617450475692749, 0.04199514910578728, 0.018608780577778816, 0.031412459909915924, 0.05092642828822136, 0.06660290062427521, 0.09416216611862183, 0.057612884789705276, 0.04849693179130554, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.29958024621009827, 0.07432066649198532, 0.03006197139620781, 0.016736475750803947, 0.038564808666706085, 0.05106877535581589, 0.09408668428659439, 0.05606791749596596, 0.054582275450229645, 0.045220401138067245, 0.056380707770586014, 0.015378053300082684, 0.04082852229475975, 0.06084694340825081, 0.030811049044132233, 0.03546451777219772, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.030439963564276695, 0.0024933936074376106, 0.005298204720020294, 0.004223393276333809, 0.005517445504665375, 0.14508457481861115, 0.01603415422141552, 0.03598647937178612, 0.0057580480352044106, 0.007398170419037342, 0.19613467156887054, 0.006090890150517225, 0.011465118266642094, 0.23737992346286774, 0.008031771518290043, 0.005861491896212101, 0.2768023610115051, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.015668945387005806, 0.0019133484456688166, 0.006747933570295572, 0.005610871594399214, 0.006710356567054987, 0.10858529061079025, 0.013436974957585335, 0.030674796551465988, 0.00869804061949253, 0.005958769004791975, 0.15696409344673157, 0.00640126271173358, 0.01435719896107912, 0.19220466911792755, 0.009938806295394897, 0.008154557086527348, 0.22861403226852417, 0.1793600618839264, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2751207649707794, 0.046356070786714554, 0.025782747194170952, 0.02037108689546585, 0.04519129917025566, 0.047415051609277725, 0.05800836905837059, 0.054494574666023254, 0.02185583859682083, 0.028674187138676643, 0.05054797977209091, 0.03578946739435196, 0.014343999326229095, 0.05505189672112465, 0.019970891997218132, 0.0250706784427166, 0.05983670800924301, 0.10591268539428711, 0.01020571868866682, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.12210460752248764, 0.029006367549300194, 0.02792610600590706, 0.030763691291213036, 0.06431955099105835, 0.05075869709253311, 0.05746712535619736, 0.03854454681277275, 0.02881445176899433, 0.03641033545136452, 0.05530688911676407, 0.02755570411682129, 0.05062380060553551, 0.06039686128497124, 0.03790910169482231, 0.034162603318691254, 0.06550325453281403, 0.08224605768918991, 0.08903136104345322, 0.011148839257657528, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.01992272026836872, 0.0015991260297596455, 0.003536272095516324, 0.0027606096118688583, 0.0036747483536601067, 0.09198574721813202, 0.009868193417787552, 0.021762730553746223, 0.0036746191326528788, 0.004800298251211643, 0.12327182292938232, 0.003948224242776632, 0.007727704476565123, 0.14901942014694214, 0.005094495136290789, 0.0038495466578751802, 0.17433811724185944, 0.1450374871492386, 0.003139176871627569, 0.0039015670772641897, 0.21708732843399048, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.025169141590595245, 0.0025891433469951153, 0.007904480211436749, 0.002818175358697772, 0.008317488245666027, 0.06284766644239426, 0.016097376123070717, 0.024833567440509796, 0.007658041547983885, 0.011317074298858643, 0.08451006561517715, 0.012135406024754047, 0.022048376500606537, 0.1001725122332573, 0.015103276818990707, 0.020152254030108452, 0.11906814575195312, 0.141489639878273, 0.006576653569936752, 0.007074056193232536, 0.1485065221786499, 0.1536109447479248, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08679644763469696, 0.01581868715584278, 0.009913153015077114, 0.005941125564277172, 0.01174909994006157, 0.04402247071266174, 0.0494115874171257, 0.047211576253175735, 0.022448517382144928, 0.02814415469765663, 0.05249372497200966, 0.013664884492754936, 0.0380445197224617, 0.05856490507721901, 0.01600872538983822, 0.028531765565276146, 0.06752399355173111, 0.1000405102968216, 0.026074493303894997, 0.01816527359187603, 0.08061447739601135, 0.14588385820388794, 0.03293200954794884, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.12203323096036911, 0.02679692953824997, 0.03471272066235542, 0.016600849106907845, 0.05098596587777138, 0.033623479306697845, 0.03376290947198868, 0.03289441019296646, 0.022763589397072792, 0.03469543159008026, 0.03710823133587837, 0.02750304900109768, 0.030508041381835938, 0.04070492461323738, 0.019409826025366783, 0.033388134092092514, 0.046037882566452026, 0.08141940832138062, 0.025792937725782394, 0.017713353037834167, 0.05326378345489502, 0.06955216079950333, 0.07340100407600403, 0.03532780334353447, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.012632706202566624, 0.003196730511263013, 0.00899962242692709, 0.003787106368690729, 0.0055902148596942425, 0.060187432914972305, 0.010173025541007519, 0.026063401252031326, 0.008280332200229168, 0.005998437758535147, 0.08480245620012283, 0.007755732629448175, 0.014807655476033688, 0.10079342871904373, 0.011636020615696907, 0.007885102182626724, 0.120613694190979, 0.12819863855838776, 0.006487111561000347, 0.00323057291097939, 0.15047506988048553, 0.10557688027620316, 0.015480604022741318, 0.01902633160352707, 0.0783216655254364, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.11176053434610367, 0.018635563552379608, 0.05622374638915062, 0.046772800385951996, 0.03736491501331329, 0.02299342304468155, 0.031365279108285904, 0.026408886536955833, 0.043359220027923584, 0.056635573506355286, 0.021786397323012352, 0.023822149261832237, 0.030641673132777214, 0.023289257660508156, 0.05430489778518677, 0.06881187111139297, 0.024489382281899452, 0.04777085781097412, 0.036271754652261734, 0.022222526371479034, 0.026998935267329216, 0.02176610752940178, 0.03442488983273506, 0.028240257874131203, 0.05891920626163483, 0.024719828739762306, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10639805346727371, 0.0371079258620739, 0.08273956179618835, 0.01785685308277607, 0.020949997007846832, 0.018305253237485886, 0.02262011356651783, 0.018192270770668983, 0.021851180121302605, 0.01251852884888649, 0.019214097410440445, 0.03034009225666523, 0.0718761533498764, 0.020787999033927917, 0.03963525965809822, 0.06151590868830681, 0.02317928522825241, 0.0439804382622242, 0.04619227349758148, 0.02066798508167267, 0.027080876752734184, 0.04386550188064575, 0.0327674001455307, 0.04369068145751953, 0.05080847069621086, 0.04029490798711777, 0.025563018396496773, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.015084554441273212, 0.003822384402155876, 0.005877268500626087, 0.0049718520604074, 0.005370147991925478, 0.05223645642399788, 0.016127675771713257, 0.03182988613843918, 0.00821112934499979, 0.011473809368908405, 0.06704685091972351, 0.010528381913900375, 0.010598219931125641, 0.07938898354768753, 0.007211714517325163, 0.0062714628875255585, 0.09142373502254486, 0.10866505652666092, 0.003973816987127066, 0.005621673539280891, 0.1098332554101944, 0.09237755835056305, 0.02296048402786255, 0.019314926117658615, 0.08864143490791321, 0.018464695662260056, 0.011022249236702919, 0.0916503295302391, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09854131937026978, 0.03967057913541794, 0.05052143707871437, 0.04794390872120857, 0.0195318553596735, 0.015683406963944435, 0.02507476694881916, 0.014997915364801884, 0.02772345580160618, 0.016418185085058212, 0.016389017924666405, 0.026316562667489052, 0.03435927629470825, 0.017757989466190338, 0.0233626626431942, 0.03763514757156372, 0.019405605271458626, 0.034421682357788086, 0.04902297630906105, 0.030190544202923775, 0.022159066051244736, 0.037540629506111145, 0.038793135434389114, 0.04032415151596069, 0.0423126220703125, 0.06829195469617844, 0.025314129889011383, 0.04330073297023773, 0.03699527308344841, 0.0, 0.0, 0.0, 0.0], [0.10688814520835876, 0.03659575805068016, 0.005083094350993633, 0.01016042660921812, 0.013297352008521557, 0.04175207018852234, 0.0262468159198761, 0.03012855537235737, 0.013482386246323586, 0.018223188817501068, 0.0458262674510479, 0.02555879019200802, 0.015618832781910896, 0.05039895698428154, 0.01165701076388359, 0.011659128591418266, 0.0551551915705204, 0.0509166419506073, 0.019206784665584564, 0.013998407870531082, 0.06383473426103592, 0.02908892184495926, 0.016710441559553146, 0.02609526738524437, 0.05484986677765846, 0.03806670755147934, 0.03896071016788483, 0.05375232174992561, 0.023679165169596672, 0.0531080923974514, 0.0, 0.0, 0.0], [0.10871604084968567, 0.06286415457725525, 0.02304510399699211, 0.012493208050727844, 0.009688262827694416, 0.02372078038752079, 0.025617338716983795, 0.028282862156629562, 0.019523099064826965, 0.015187123790383339, 0.024615760892629623, 0.018855812028050423, 0.013071928173303604, 0.02569747343659401, 0.010639789514243603, 0.007336440030485392, 0.028488244861364365, 0.040875114500522614, 0.023693937808275223, 0.02846083790063858, 0.031781155616045, 0.03399210050702095, 0.01690451242029667, 0.04699751362204552, 0.054809220135211945, 0.0604817196726799, 0.021137842908501625, 0.05147771164774895, 0.01530434750020504, 0.07868867367506027, 0.03755194693803787, 0.0, 0.0], [0.16510623693466187, 0.013307320885360241, 0.022820336744189262, 0.032224975526332855, 0.028327813372015953, 0.011912115849554539, 0.021759122610092163, 0.012591322883963585, 0.025371331721544266, 0.01592349074780941, 0.011745063588023186, 0.021340155974030495, 0.022658517584204674, 0.012096998281776905, 0.05877390503883362, 0.041142962872982025, 0.012866133823990822, 0.02189669944345951, 0.02738575078547001, 0.015412718988955021, 0.014822259545326233, 0.013198777101933956, 0.029060106724500656, 0.04031305015087128, 0.023036926984786987, 0.05035790055990219, 0.046500690281391144, 0.03221071884036064, 0.04574136063456535, 0.011448168195784092, 0.05148483067750931, 0.04716223478317261, 0.0], [0.019862741231918335, 0.0006941381143406034, 0.0017670642118901014, 0.0019038202008232474, 0.0023115782532840967, 0.05924388766288757, 0.005164953414350748, 0.008714546449482441, 0.0024906685575842857, 0.0030579573940485716, 0.07585158944129944, 0.002390549285337329, 0.0031773503869771957, 0.08945121616125107, 0.0030652948189526796, 0.0016209169989451766, 0.10312031954526901, 0.0625915676355362, 0.002058295765891671, 0.002232219558209181, 0.12742049992084503, 0.024703850969672203, 0.0035101925022900105, 0.00499417819082737, 0.028317121788859367, 0.005874201189726591, 0.006215325556695461, 0.03686564415693283, 0.005438121035695076, 0.004061868414282799, 0.004735523369163275, 0.004534382838755846, 0.2925584614276886]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9726406335830688, 0.027359364554286003, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8850566744804382, 0.08160645514726639, 0.03333686292171478, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7844979166984558, 0.09045999497175217, 0.10720972716808319, 0.01783234067261219, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6679065227508545, 0.10135099291801453, 0.10997921228408813, 0.0699087604880333, 0.050854478031396866, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.586279034614563, 0.06960930675268173, 0.06798022985458374, 0.06058962643146515, 0.08339767158031464, 0.13214409351348877, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5015710592269897, 0.06227877363562584, 0.07209988683462143, 0.05590919405221939, 0.08146880567073822, 0.12033165246248245, 0.10634062439203262, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.40016087889671326, 0.05525398626923561, 0.06347683072090149, 0.05413786694407463, 0.07119617611169815, 0.11230527609586716, 0.12489721924066544, 0.11857166141271591, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4018532931804657, 0.08579416573047638, 0.056448839604854584, 0.07181122153997421, 0.08318912237882614, 0.07845646142959595, 0.09191881120204926, 0.07768876850605011, 0.05283927917480469, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3827159106731415, 0.07976946234703064, 0.062077559530735016, 0.06360235810279846, 0.08211023360490799, 0.07804954797029495, 0.08597437292337418, 0.07506486773490906, 0.0632784441113472, 0.02735721692442894, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3404167592525482, 0.04535197094082832, 0.04658910632133484, 0.0403917022049427, 0.05489042401313782, 0.08179006725549698, 0.08906516432762146, 0.09645897150039673, 0.05564969405531883, 0.06398677080869675, 0.08540938794612885, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.31186187267303467, 0.09539996832609177, 0.07292013615369797, 0.0697687417268753, 0.07777827978134155, 0.06735038757324219, 0.06161406263709068, 0.054810743778944016, 0.052520137280225754, 0.05022520199418068, 0.07299348711967468, 0.012756960466504097, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.36652863025665283, 0.06308672577142715, 0.03947845473885536, 0.04609525203704834, 0.05740750953555107, 0.0634833425283432, 0.05238465964794159, 0.05501618608832359, 0.04584015905857086, 0.06228693574666977, 0.0658990740776062, 0.058063607662916183, 0.024429509416222572, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.28564438223838806, 0.03804025053977966, 0.03942390903830528, 0.034238625317811966, 0.0463939793407917, 0.06771662831306458, 0.07328074425458908, 0.07855433225631714, 0.047896627336740494, 0.055936526507139206, 0.07124947756528854, 0.035689257085323334, 0.04956042021512985, 0.07637479156255722, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.297754168510437, 0.05259547755122185, 0.04347941279411316, 0.05845862627029419, 0.04568730294704437, 0.05259405076503754, 0.04456199333071709, 0.0459747239947319, 0.06316147744655609, 0.0776875764131546, 0.05499143525958061, 0.03466471657156944, 0.05020103231072426, 0.05886140838265419, 0.01932653971016407, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.28853145241737366, 0.04616911709308624, 0.028233066201210022, 0.038242388516664505, 0.04330621287226677, 0.05174916982650757, 0.04318275675177574, 0.043931201100349426, 0.05616341903805733, 0.06773821264505386, 0.05490177869796753, 0.08139309287071228, 0.04325690492987633, 0.0590481236577034, 0.03989904746413231, 0.014254068955779076, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.23721373081207275, 0.03161581605672836, 0.03324565663933754, 0.029231945052742958, 0.038618117570877075, 0.05549168214201927, 0.05966496840119362, 0.06321781128644943, 0.040736980736255646, 0.04752674326300621, 0.058378204703330994, 0.03039967454969883, 0.04287158325314522, 0.06284616142511368, 0.053212910890579224, 0.04736898094415665, 0.06835903972387314, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2098696380853653, 0.02729126438498497, 0.031156614422798157, 0.026615004986524582, 0.03165113925933838, 0.05273226648569107, 0.052745021879673004, 0.05635902285575867, 0.04076546058058739, 0.04278237372636795, 0.056191232055425644, 0.030678272247314453, 0.04398961737751961, 0.06091473996639252, 0.0501369833946228, 0.0415731742978096, 0.06652864068746567, 0.07801953703165054, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2385382503271103, 0.04414580762386322, 0.046435050666332245, 0.0345127172768116, 0.0357217937707901, 0.04369329661130905, 0.04304974898695946, 0.034988246858119965, 0.043718066066503525, 0.0395219661295414, 0.044578537344932556, 0.024599451571702957, 0.0625118762254715, 0.04753483459353447, 0.043756093829870224, 0.04995930567383766, 0.050908081233501434, 0.0526624359190464, 0.019164469093084335, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.17729151248931885, 0.042214617133140564, 0.030818725004792213, 0.030886096879839897, 0.033868495374917984, 0.045211877673864365, 0.042901039123535156, 0.032215580344200134, 0.040188658982515335, 0.049268968403339386, 0.04801624268293381, 0.03814242035150528, 0.051265135407447815, 0.05206841975450516, 0.047441162168979645, 0.03924533724784851, 0.05630478635430336, 0.05462942644953728, 0.0599186010658741, 0.02810293436050415, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.19048276543617249, 0.025203485041856766, 0.026787471026182175, 0.02367268316447735, 0.03110402822494507, 0.04345344379544258, 0.04649331048130989, 0.048206478357315063, 0.03323280066251755, 0.03892753645777702, 0.04549149051308632, 0.02444273605942726, 0.035701580345630646, 0.04907767102122307, 0.04303564876317978, 0.039055656641721725, 0.0537174753844738, 0.06658957898616791, 0.04320826008915901, 0.03176470100879669, 0.06035127118229866, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14975358545780182, 0.023474570363759995, 0.025508398190140724, 0.028942441567778587, 0.03517158329486847, 0.038801901042461395, 0.04594280198216438, 0.03962729126214981, 0.037087779492139816, 0.038723014295101166, 0.04190719500184059, 0.021688053384423256, 0.04328255355358124, 0.04592541232705116, 0.03893951326608658, 0.0400761142373085, 0.05060023441910744, 0.060830533504486084, 0.043721601366996765, 0.03023521602153778, 0.05728611722588539, 0.062474027276039124, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.17513407766819, 0.027519281953573227, 0.029514014720916748, 0.02071532793343067, 0.026873527094721794, 0.0345461331307888, 0.038530781865119934, 0.033037811517715454, 0.04181556776165962, 0.04010028764605522, 0.03696895390748978, 0.030468937009572983, 0.039953943341970444, 0.04023702070116997, 0.0448262095451355, 0.03708262741565704, 0.04420144855976105, 0.04931037127971649, 0.04125744849443436, 0.028610678389668465, 0.04997344687581062, 0.05269285663962364, 0.036629173904657364, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.13285990059375763, 0.040708061307668686, 0.04111678898334503, 0.034844279289245605, 0.04087604954838753, 0.027567215263843536, 0.028293700888752937, 0.022520417347550392, 0.0387541837990284, 0.040378037840127945, 0.029089409857988358, 0.029609227553009987, 0.0567011833190918, 0.031181858852505684, 0.037661172449588776, 0.056886378675699234, 0.03396708890795708, 0.037813056260347366, 0.04951409250497818, 0.031945571303367615, 0.037599027156829834, 0.04326637089252472, 0.043056681752204895, 0.033790234476327896, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14709368348121643, 0.018218709155917168, 0.022143620997667313, 0.02162139303982258, 0.02707289345562458, 0.03385418280959129, 0.03787611797451973, 0.03801965340971947, 0.027932504191994667, 0.03090183064341545, 0.03606922924518585, 0.01902313157916069, 0.03465801104903221, 0.03916986659169197, 0.033755578100681305, 0.02613663300871849, 0.042978327721357346, 0.05313742905855179, 0.03463505581021309, 0.02014029398560524, 0.04878358915448189, 0.04990803450345993, 0.04624392092227936, 0.04715671017765999, 0.06346960365772247, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1760278195142746, 0.01488215196877718, 0.026751182973384857, 0.0240880586206913, 0.03566748648881912, 0.027094293385744095, 0.02498425543308258, 0.021471885964274406, 0.026359373703598976, 0.023845119401812553, 0.027566982433199883, 0.01686953380703926, 0.0777742937207222, 0.029109211638569832, 0.04799698293209076, 0.02900250256061554, 0.03132336959242821, 0.03615599498152733, 0.03959561511874199, 0.017008014023303986, 0.034839075058698654, 0.03443719446659088, 0.04149218648672104, 0.09124380350112915, 0.040522992610931396, 0.0038906026165932417, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1850174218416214, 0.027787016704678535, 0.025097640231251717, 0.027018999680876732, 0.029199207201600075, 0.030337024480104446, 0.032221775501966476, 0.02301507070660591, 0.0267928596585989, 0.022793851792812347, 0.030674852430820465, 0.021220838651061058, 0.053888075053691864, 0.03259299322962761, 0.030737651512026787, 0.025725245475769043, 0.03486606478691101, 0.037935830652713776, 0.028672009706497192, 0.026148982346057892, 0.03875335678458214, 0.04179506003856659, 0.03863157331943512, 0.05337152257561684, 0.03982071578502655, 0.02571161463856697, 0.010172729380428791, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.12446325272321701, 0.01676889881491661, 0.02010510489344597, 0.019267024472355843, 0.024457814171910286, 0.03020336478948593, 0.03127393499016762, 0.03254607319831848, 0.023186003789305687, 0.02350839227437973, 0.0317331999540329, 0.017080510035157204, 0.029082054272294044, 0.03428216651082039, 0.03579127788543701, 0.025820568203926086, 0.037531185895204544, 0.04618249833583832, 0.029500003904104233, 0.017913807183504105, 0.04274731129407883, 0.042434703558683395, 0.039002396166324615, 0.04141838476061821, 0.05949503183364868, 0.03595735505223274, 0.030984872952103615, 0.057262834161520004, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14491021633148193, 0.025843990966677666, 0.031176971271634102, 0.01946275867521763, 0.012575727887451649, 0.025274591520428658, 0.022020690143108368, 0.021267279982566833, 0.03165539726614952, 0.02766416408121586, 0.026251589879393578, 0.022091038525104523, 0.03851794824004173, 0.028266867622733116, 0.0288776233792305, 0.036865364760160446, 0.03069997765123844, 0.03204502537846565, 0.03411627188324928, 0.02035541459918022, 0.03446315601468086, 0.03542865440249443, 0.030728410929441452, 0.04441893845796585, 0.03793596476316452, 0.05734865367412567, 0.03889700025320053, 0.04111205041408539, 0.019728200510144234, 0.0, 0.0, 0.0, 0.0], [0.13168425858020782, 0.025460105389356613, 0.02796240895986557, 0.024109845981001854, 0.01686880923807621, 0.031406790018081665, 0.023247336968779564, 0.022071687504649162, 0.027362508699297905, 0.02637045457959175, 0.032045334577560425, 0.02877580001950264, 0.04614202678203583, 0.03455977141857147, 0.03132109344005585, 0.02943773753941059, 0.03749070689082146, 0.03597421571612358, 0.0306710172444582, 0.026778314262628555, 0.041662365198135376, 0.03274398297071457, 0.030348768457770348, 0.032651983201503754, 0.038056500256061554, 0.034575607627630234, 0.022936122491955757, 0.03690328821539879, 0.02596619911491871, 0.01441489439457655, 0.0, 0.0, 0.0], [0.1656370609998703, 0.02280924655497074, 0.029107755050063133, 0.02494465932250023, 0.02435287833213806, 0.02503182739019394, 0.018486661836504936, 0.01857367716729641, 0.028443630784749985, 0.029199941083788872, 0.0253182053565979, 0.03220498561859131, 0.025753282010555267, 0.026890520006418228, 0.031273938715457916, 0.03305148705840111, 0.028835458680987358, 0.029910407960414886, 0.0275588259100914, 0.024624649435281754, 0.0320536270737648, 0.02649376355111599, 0.02170960046350956, 0.031640153378248215, 0.03707476332783699, 0.0479947030544281, 0.02793215773999691, 0.034756116569042206, 0.03804944083094597, 0.02153567597270012, 0.008750851266086102, 0.0, 0.0], [0.16794215142726898, 0.025983478873968124, 0.03222338482737541, 0.019469385966658592, 0.023735277354717255, 0.023115742951631546, 0.0199736300855875, 0.016550231724977493, 0.029796775430440903, 0.028494667261838913, 0.023108843713998795, 0.02208707109093666, 0.042165566235780716, 0.02439018338918686, 0.02074914425611496, 0.038735877722501755, 0.026269812136888504, 0.02728910744190216, 0.019617389887571335, 0.020242862403392792, 0.02920321375131607, 0.030413366854190826, 0.02576262876391411, 0.040849827229976654, 0.033326659351587296, 0.021795300766825676, 0.026310516521334648, 0.03101349249482155, 0.03689264506101608, 0.01937715709209442, 0.04355606809258461, 0.009558655321598053, 0.0], [0.11396828293800354, 0.01775287464261055, 0.01606196165084839, 0.015585055574774742, 0.01839612051844597, 0.024367082864046097, 0.026053881272673607, 0.023999109864234924, 0.021919487044215202, 0.026251113042235374, 0.024730533361434937, 0.015323017723858356, 0.025614570826292038, 0.0263378843665123, 0.024982867762446404, 0.02464262954890728, 0.028590328991413116, 0.03295150771737099, 0.027568619698286057, 0.01848754845559597, 0.03220851346850395, 0.031914692372083664, 0.036307260394096375, 0.0341540202498436, 0.036785583943128586, 0.03297772631049156, 0.026278911158442497, 0.03809891268610954, 0.029801947996020317, 0.018183261156082153, 0.03632068261504173, 0.035680945962667465, 0.05770310387015343]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8058414459228516, 0.19415856897830963, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6373865008354187, 0.09672051668167114, 0.26589304208755493, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5870265364646912, 0.09922721236944199, 0.07913831621408463, 0.23460792005062103, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4194737374782562, 0.06312593817710876, 0.08123067021369934, 0.08381500095129013, 0.35235461592674255, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5308531522750854, 0.0959072858095169, 0.07779339700937271, 0.06661548465490341, 0.07293295115232468, 0.15589767694473267, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.38503071665763855, 0.07478094846010208, 0.06978150457143784, 0.047606538981199265, 0.051449161022901535, 0.08776707202196121, 0.2835841178894043, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2962956726551056, 0.05328270420432091, 0.06944749504327774, 0.04097547382116318, 0.06660831719636917, 0.1021856963634491, 0.10869503766298294, 0.26250961422920227, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.30474990606307983, 0.06337985396385193, 0.05753423646092415, 0.04057719185948372, 0.0730014219880104, 0.07375827431678772, 0.07239888608455658, 0.05693604797124863, 0.25766414403915405, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.29880762100219727, 0.05552824214100838, 0.05443597584962845, 0.05285302922129631, 0.06568750739097595, 0.07554222643375397, 0.06968103349208832, 0.054113879799842834, 0.09120790660381317, 0.18214264512062073, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.26933014392852783, 0.06352431327104568, 0.059521228075027466, 0.052424702793359756, 0.06052157282829285, 0.12185832113027573, 0.07608654350042343, 0.09753072261810303, 0.04232607036828995, 0.05058327317237854, 0.10629311949014664, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.22074301540851593, 0.06960077583789825, 0.048723481595516205, 0.06675717234611511, 0.06954211741685867, 0.06483786553144455, 0.046523742377758026, 0.039300836622714996, 0.018016092479228973, 0.0358063168823719, 0.053022876381874084, 0.26712578535079956, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15080073475837708, 0.04732475057244301, 0.12551115453243256, 0.06240082159638405, 0.05354631692171097, 0.05578393489122391, 0.04542611539363861, 0.03888600319623947, 0.0301998108625412, 0.033370546996593475, 0.04645953327417374, 0.023437919095158577, 0.2868523597717285, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2044883519411087, 0.04972049966454506, 0.04838546738028526, 0.043853580951690674, 0.05115717276930809, 0.1007489338517189, 0.06420683115720749, 0.08302561193704605, 0.03880453109741211, 0.047957729548215866, 0.09582382440567017, 0.03239700198173523, 0.04441104456782341, 0.0950193703174591, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1623673439025879, 0.028519174084067345, 0.047843094915151596, 0.041354816406965256, 0.040284883230924606, 0.0520053468644619, 0.04226505383849144, 0.03821168839931488, 0.05986882373690605, 0.06849399209022522, 0.04734278842806816, 0.019715160131454468, 0.06202228367328644, 0.04572192579507828, 0.24398362636566162, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.13128933310508728, 0.039628103375434875, 0.059519413858652115, 0.03966323286294937, 0.04406355693936348, 0.04348496347665787, 0.03726547583937645, 0.03240034729242325, 0.03720574453473091, 0.06481601297855377, 0.040474534034729004, 0.025914235040545464, 0.06507598608732224, 0.03922511264681816, 0.07744196802377701, 0.2225320190191269, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15813341736793518, 0.03911251947283745, 0.03856496512889862, 0.03587968647480011, 0.04187794774770737, 0.08161000907421112, 0.05245806276798248, 0.0681268721818924, 0.03444807231426239, 0.04342752322554588, 0.08275996893644333, 0.02933340333402157, 0.041573796421289444, 0.08478686958551407, 0.04881599545478821, 0.032352838665246964, 0.08673813939094543, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.13890595734119415, 0.03169632703065872, 0.03251556307077408, 0.03538898378610611, 0.038000334054231644, 0.06974425166845322, 0.059424545615911484, 0.06768239289522171, 0.027843810617923737, 0.039054665714502335, 0.07273104041814804, 0.030109861865639687, 0.04163092002272606, 0.07565011829137802, 0.040188319981098175, 0.027483616024255753, 0.07795906811952591, 0.09399020671844482, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09691290557384491, 0.030687319114804268, 0.030817966908216476, 0.04473433643579483, 0.04171944037079811, 0.036036767065525055, 0.04101703688502312, 0.032104767858982086, 0.04344063252210617, 0.048855531960725784, 0.03457901254296303, 0.016025088727474213, 0.049941662698984146, 0.0346352756023407, 0.060656867921352386, 0.034977883100509644, 0.03454342484474182, 0.037827931344509125, 0.25048619508743286, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.12059216946363449, 0.04121065139770508, 0.0509851798415184, 0.052891526371240616, 0.02886129543185234, 0.04706199839711189, 0.0292128287255764, 0.038170017302036285, 0.021046487614512444, 0.029787806794047356, 0.04406721517443657, 0.02650333382189274, 0.04127003997564316, 0.04430737346410751, 0.03080155700445175, 0.04064903408288956, 0.044745106250047684, 0.042122457176446915, 0.02844228595495224, 0.1972716897726059, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.11388304829597473, 0.028255397453904152, 0.028351297602057457, 0.026958664879202843, 0.031310852617025375, 0.05977548286318779, 0.0391821525990963, 0.05072040110826492, 0.027684014290571213, 0.035564228892326355, 0.06452134251594543, 0.02426709607243538, 0.03616495057940483, 0.06838168203830719, 0.04139300435781479, 0.029094640165567398, 0.0723004937171936, 0.07349380850791931, 0.037000205367803574, 0.0352863185107708, 0.07641094923019409, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08453849703073502, 0.017416605725884438, 0.01894914172589779, 0.0165606327354908, 0.02229611948132515, 0.04174060747027397, 0.05894864723086357, 0.03893362358212471, 0.030214650556445122, 0.034642331302165985, 0.0483190156519413, 0.016358422115445137, 0.02567455545067787, 0.05309673771262169, 0.02054651454091072, 0.02046983130276203, 0.057046402245759964, 0.06604909896850586, 0.03774644806981087, 0.021413879469037056, 0.06129930540919304, 0.2077389657497406, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1124843955039978, 0.024394847452640533, 0.024523155763745308, 0.027489561587572098, 0.026104671880602837, 0.03731505200266838, 0.06748531758785248, 0.03890375792980194, 0.01798143796622753, 0.028454387560486794, 0.03756532073020935, 0.01832606829702854, 0.020088663324713707, 0.03865170478820801, 0.0232856422662735, 0.024226617068052292, 0.03960612043738365, 0.04833188280463219, 0.04500093311071396, 0.017362147569656372, 0.04009140655398369, 0.05208287015557289, 0.19024400413036346, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08183357119560242, 0.019828449934720993, 0.02687339298427105, 0.029252557083964348, 0.02518443763256073, 0.03192773088812828, 0.029049793258309364, 0.024972520768642426, 0.03350626677274704, 0.04007429629564285, 0.033517319709062576, 0.016035988926887512, 0.03204454854130745, 0.03448064625263214, 0.028517093509435654, 0.03532731533050537, 0.03583228215575218, 0.04058396443724632, 0.029941106215119362, 0.02109791524708271, 0.03671329841017723, 0.039212584495544434, 0.03543241322040558, 0.23876062035560608, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10145692527294159, 0.014237047173082829, 0.016984183341264725, 0.018636571243405342, 0.02853623777627945, 0.0365380235016346, 0.02957051247358322, 0.041983190923929214, 0.014166045933961868, 0.030081408098340034, 0.04016641527414322, 0.02007502317428589, 0.030370114371180534, 0.043534401804208755, 0.030393019318580627, 0.020717334002256393, 0.04677111282944679, 0.0549510233104229, 0.028809182345867157, 0.020043687894940376, 0.05092543363571167, 0.04084467142820358, 0.0418580137193203, 0.03848976269364357, 0.15986064076423645, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.07937924563884735, 0.020674310624599457, 0.020799757912755013, 0.027988232672214508, 0.04565475508570671, 0.02656259387731552, 0.02539670839905739, 0.021243257448077202, 0.01776379905641079, 0.02520064264535904, 0.027576463297009468, 0.030607089400291443, 0.034380000084638596, 0.028568314388394356, 0.022726934403181076, 0.01605195552110672, 0.029712749645113945, 0.03276180848479271, 0.025702400133013725, 0.012878884561359882, 0.030626127496361732, 0.02036965638399124, 0.03741392120718956, 0.027850378304719925, 0.035718515515327454, 0.27639153599739075, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09605434536933899, 0.02309289574623108, 0.024199659004807472, 0.024974144995212555, 0.036829639226198196, 0.03001439943909645, 0.021069226786494255, 0.018117526546120644, 0.022569267079234123, 0.02834322676062584, 0.030505143105983734, 0.021707680076360703, 0.028211787343025208, 0.031639426946640015, 0.03916612267494202, 0.03627365827560425, 0.03311571851372719, 0.03514396771788597, 0.02574523724615574, 0.015889015048742294, 0.034675464034080505, 0.02677008882164955, 0.027143454179167747, 0.0404508002102375, 0.037603847682476044, 0.032323356717824936, 0.17837099730968475, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.07476937770843506, 0.011021512560546398, 0.010706555098295212, 0.015325457789003849, 0.01990356110036373, 0.03285866603255272, 0.021936072036623955, 0.02945413999259472, 0.01642150804400444, 0.022476578131318092, 0.03722343221306801, 0.01758023351430893, 0.021954145282506943, 0.040462132543325424, 0.031415410339832306, 0.010417776182293892, 0.044297393411397934, 0.05118127167224884, 0.029565559700131416, 0.015394334681332111, 0.0490642674267292, 0.03202195093035698, 0.030905120074748993, 0.04132053256034851, 0.06578273326158524, 0.03978404775261879, 0.02049970254302025, 0.16625656187534332, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05124205723404884, 0.011390028521418571, 0.02026580646634102, 0.024529647082090378, 0.12732437252998352, 0.020083913579583168, 0.017692744731903076, 0.020381992682814598, 0.018548619002103806, 0.023206066340208054, 0.021026061847805977, 0.020521236583590508, 0.025210734456777573, 0.021994242444634438, 0.017190473154187202, 0.02080591395497322, 0.023035308346152306, 0.026458054780960083, 0.031086130067706108, 0.011454472318291664, 0.024357995018363, 0.02252495475113392, 0.024600783362984657, 0.024588001891970634, 0.02838081493973732, 0.04660612717270851, 0.043627094477415085, 0.02030046284198761, 0.21156589686870575, 0.0, 0.0, 0.0, 0.0], [0.07929293811321259, 0.023537395521998405, 0.022847289219498634, 0.01993604749441147, 0.015462876297533512, 0.025110295042395592, 0.021429050713777542, 0.015073496848344803, 0.00992409698665142, 0.015724744647741318, 0.027188707143068314, 0.020747287198901176, 0.03497635945677757, 0.029102325439453125, 0.021219033747911453, 0.01814761571586132, 0.03105819784104824, 0.03380950167775154, 0.020137565210461617, 0.012323119677603245, 0.0336926244199276, 0.02433612570166588, 0.031084517017006874, 0.03377701714634895, 0.04355444014072418, 0.032963525503873825, 0.03314930200576782, 0.02536490187048912, 0.022163692861795425, 0.2228659689426422, 0.0, 0.0, 0.0], [0.10059655457735062, 0.03096376359462738, 0.025578510016202927, 0.0202240701764822, 0.027132991701364517, 0.0294907595962286, 0.0407448373734951, 0.021403610706329346, 0.016392609104514122, 0.02252069301903248, 0.027555176988244057, 0.013599206693470478, 0.027306707575917244, 0.027806581929326057, 0.011214177124202251, 0.023586539551615715, 0.02839917503297329, 0.03066891059279442, 0.021664204075932503, 0.017032310366630554, 0.02905159257352352, 0.026549918577075005, 0.03297552466392517, 0.026790324598550797, 0.032060641795396805, 0.02729833871126175, 0.017307184636592865, 0.018768811598420143, 0.028226524591445923, 0.017085110768675804, 0.18000467121601105, 0.0, 0.0], [0.08361606299877167, 0.016581889241933823, 0.015823302790522575, 0.022005166858434677, 0.02406255342066288, 0.021028300747275352, 0.022811293601989746, 0.014412387274205685, 0.01557208877056837, 0.015104830265045166, 0.020917734131217003, 0.021798150613904, 0.02598598226904869, 0.02201320417225361, 0.022025059908628464, 0.018091218546032906, 0.02311510592699051, 0.025451799854636192, 0.02594352327287197, 0.011241885833442211, 0.024466173723340034, 0.0228698942810297, 0.031613439321517944, 0.029542673379182816, 0.02381790056824684, 0.029349016025662422, 0.025922643020749092, 0.017268173396587372, 0.0318446047604084, 0.012037734501063824, 0.027524925768375397, 0.2561413049697876, 0.0], [0.05258440971374512, 0.012508383020758629, 0.012737675569951534, 0.012144193053245544, 0.0144678158685565, 0.024812448769807816, 0.01710914634168148, 0.02516256831586361, 0.016298966482281685, 0.015160080045461655, 0.029820043593645096, 0.014290276914834976, 0.024094797670841217, 0.033742766827344894, 0.021632254123687744, 0.018924105912446976, 0.03819216787815094, 0.04115181416273117, 0.024165764451026917, 0.020635638386011124, 0.044185325503349304, 0.03793709725141525, 0.029636353254318237, 0.03152839466929436, 0.04542306438088417, 0.035963598638772964, 0.030482539907097816, 0.04609914869070053, 0.03206766024231911, 0.01766294240951538, 0.03169601783156395, 0.04462910816073418, 0.10305341333150864]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7707713842391968, 0.22922858595848083, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7813623547554016, 0.109785296022892, 0.10885230451822281, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.535038948059082, 0.11194943636655807, 0.15013694763183594, 0.20287470519542694, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.41803574562072754, 0.1603638231754303, 0.1074029952287674, 0.10625781118869781, 0.20793959498405457, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4349866509437561, 0.09579858183860779, 0.08251801878213882, 0.10028504580259323, 0.15332908928394318, 0.1330825835466385, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.44621357321739197, 0.05624214559793472, 0.05040299892425537, 0.07084295898675919, 0.09360906481742859, 0.11472620815038681, 0.16796305775642395, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3609195351600647, 0.07294464111328125, 0.05191005766391754, 0.07159046083688736, 0.09321668744087219, 0.09543665498495102, 0.14539918303489685, 0.10858277231454849, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3171406686306, 0.07683130353689194, 0.05552283674478531, 0.06445366144180298, 0.08586590737104416, 0.10643255710601807, 0.1452113687992096, 0.09439337253570557, 0.05414833128452301, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.19288603961467743, 0.1134386956691742, 0.04530505836009979, 0.08361203223466873, 0.05763273686170578, 0.11914196610450745, 0.11441227793693542, 0.12364513427019119, 0.07640958577394485, 0.07351639866828918, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.25174087285995483, 0.060194116085767746, 0.04930208995938301, 0.059223245829343796, 0.08453336358070374, 0.07307681441307068, 0.12308554351329803, 0.08293969184160233, 0.06380272656679153, 0.07185835391283035, 0.08024313300848007, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2479359656572342, 0.048557743430137634, 0.061716966331005096, 0.06616783887147903, 0.0878860130906105, 0.05920383334159851, 0.09128588438034058, 0.06866813451051712, 0.060628652572631836, 0.08399352431297302, 0.0656941756606102, 0.05826116353273392, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15039603412151337, 0.0707230269908905, 0.036188021302223206, 0.03812884911894798, 0.05496574938297272, 0.09552797675132751, 0.1278020143508911, 0.09101750701665878, 0.04133947193622589, 0.06619426608085632, 0.11493087559938431, 0.03526817262172699, 0.0775180533528328, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.21730750799179077, 0.052248772233724594, 0.04279686510562897, 0.05130988731980324, 0.07021071761846542, 0.06084864214062691, 0.10094168037176132, 0.06729137152433395, 0.05347270891070366, 0.05803239718079567, 0.063866026699543, 0.0414368212223053, 0.05429811403155327, 0.06593842804431915, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14158985018730164, 0.08735613524913788, 0.041096266359090805, 0.07771317660808563, 0.0664159283041954, 0.06084612384438515, 0.07890869677066803, 0.06467185169458389, 0.041567541658878326, 0.05314023420214653, 0.060331959277391434, 0.049903471022844315, 0.06147686019539833, 0.06152055785059929, 0.053461313247680664, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.11277392506599426, 0.05140271782875061, 0.03480074182152748, 0.044109929352998734, 0.04927106201648712, 0.08066023141145706, 0.09371130168437958, 0.07634277641773224, 0.03460073471069336, 0.0555649995803833, 0.09000370651483536, 0.027950923889875412, 0.058086883276700974, 0.09759874641895294, 0.058530546724796295, 0.03459078073501587, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.19136789441108704, 0.04588800296187401, 0.037184182554483414, 0.04478847235441208, 0.058455415070056915, 0.0516231432557106, 0.08451797068119049, 0.056617289781570435, 0.04477924853563309, 0.04681586101651192, 0.05177522450685501, 0.034743115305900574, 0.04365590214729309, 0.05257752537727356, 0.045719120651483536, 0.05523938685655594, 0.05425224080681801, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.178098663687706, 0.03651316463947296, 0.035378020256757736, 0.04660060256719589, 0.056067463010549545, 0.05542533099651337, 0.06580902636051178, 0.058695923537015915, 0.04460207000374794, 0.03845333680510521, 0.05522071197628975, 0.02812928520143032, 0.038876280188560486, 0.056352097541093826, 0.04307202994823456, 0.048592109233140945, 0.05801301822066307, 0.056100811809301376, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.11277126520872116, 0.04416341334581375, 0.01679084822535515, 0.03922899439930916, 0.030570238828659058, 0.08429381251335144, 0.05829833820462227, 0.07985574752092361, 0.027740228921175003, 0.024607932195067406, 0.0856802761554718, 0.01213553361594677, 0.021223612129688263, 0.08979952335357666, 0.02439562790095806, 0.018000086769461632, 0.0979786366224289, 0.08564859628677368, 0.04681731015443802, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15648671984672546, 0.043593376874923706, 0.029469136148691177, 0.02338278852403164, 0.03989394009113312, 0.06601448357105255, 0.07239893823862076, 0.049598775804042816, 0.025720087811350822, 0.028729889541864395, 0.07268749177455902, 0.020104380324482918, 0.038949914276599884, 0.0779348835349083, 0.031850170344114304, 0.029572613537311554, 0.08162925392389297, 0.04983029142022133, 0.04757083207368851, 0.014582027681171894, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.16675154864788055, 0.040216755121946335, 0.031946271657943726, 0.03915555030107498, 0.04867977276444435, 0.043519362807273865, 0.07027734071016312, 0.04676978662610054, 0.03737045079469681, 0.03785951808094978, 0.04185349866747856, 0.029438791796565056, 0.03496702387928963, 0.04187152162194252, 0.036659594625234604, 0.04418949410319328, 0.04244700446724892, 0.04308999702334404, 0.055408261716365814, 0.02402997761964798, 0.04349849000573158, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.21311703324317932, 0.03452147915959358, 0.026512037962675095, 0.037284668534994125, 0.04200230911374092, 0.05378075689077377, 0.06410719454288483, 0.040195900946855545, 0.029513953253626823, 0.030562065541744232, 0.048140499740839005, 0.019645733758807182, 0.023855963721871376, 0.04712359979748726, 0.027805231511592865, 0.025632821023464203, 0.04690323397517204, 0.042114999145269394, 0.04245338588953018, 0.016209973022341728, 0.04720157012343407, 0.041315577924251556, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.16099101305007935, 0.03149211406707764, 0.025505999103188515, 0.03203628212213516, 0.038119371980428696, 0.05386463925242424, 0.05289200320839882, 0.06297367811203003, 0.03872942551970482, 0.03070300817489624, 0.04861500486731529, 0.016964146867394447, 0.020984871312975883, 0.04743487387895584, 0.026178555563092232, 0.032817039638757706, 0.048741020262241364, 0.03702303394675255, 0.05028446391224861, 0.014425938948988914, 0.04926920309662819, 0.02748514525592327, 0.052469104528427124, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.13924191892147064, 0.04732196778059006, 0.022920068353414536, 0.037826746702194214, 0.03733159974217415, 0.056281980127096176, 0.06773107498884201, 0.0638374611735344, 0.028824560344219208, 0.028368612751364708, 0.05008407309651375, 0.014775178395211697, 0.021058905869722366, 0.04819713160395622, 0.017685124650597572, 0.01545622292906046, 0.0484280101954937, 0.037099532783031464, 0.042473260313272476, 0.01842682436108589, 0.04861331358551979, 0.022673960775136948, 0.05998954549431801, 0.02535295858979225, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.195623517036438, 0.03233327716588974, 0.022874081507325172, 0.038390763103961945, 0.03449878469109535, 0.04616905376315117, 0.055537495762109756, 0.03960668295621872, 0.02309839427471161, 0.024249285459518433, 0.04043305665254593, 0.016078554093837738, 0.021624945104122162, 0.03903055936098099, 0.026520881801843643, 0.028113437816500664, 0.03823632746934891, 0.035426102578639984, 0.04174575209617615, 0.02109713852405548, 0.03767955303192139, 0.037286918610334396, 0.04423920437693596, 0.022921495139598846, 0.037184685468673706, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0581965334713459, 0.038897398859262466, 0.02600524015724659, 0.02356790192425251, 0.022979989647865295, 0.053841520100831985, 0.058061059564352036, 0.06858771294355392, 0.037459827959537506, 0.029159093275666237, 0.05474674701690674, 0.013592950068414211, 0.027647046372294426, 0.05575403571128845, 0.030779868364334106, 0.01613735221326351, 0.05745428055524826, 0.03968728333711624, 0.03406553715467453, 0.013110978528857231, 0.059776563197374344, 0.016043761745095253, 0.029699290171265602, 0.020566539838910103, 0.08347290754318237, 0.03070860728621483, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15561480820178986, 0.030323194339871407, 0.02106756716966629, 0.038600414991378784, 0.025089262053370476, 0.04719521105289459, 0.04255649819970131, 0.051662590354681015, 0.030368102714419365, 0.020133620128035545, 0.04255696386098862, 0.018511781468987465, 0.01867171935737133, 0.0414302833378315, 0.022022131830453873, 0.02106102742254734, 0.0416661836206913, 0.027249205857515335, 0.04048159345984459, 0.015536420047283173, 0.04219621419906616, 0.017007341608405113, 0.06220947951078415, 0.015703151002526283, 0.05443187430500984, 0.04092409461736679, 0.015729239210486412, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1521887332201004, 0.03232048079371452, 0.026277603581547737, 0.03215136379003525, 0.03708053380250931, 0.03843334689736366, 0.042185209691524506, 0.034409213811159134, 0.029296185821294785, 0.02475224994122982, 0.03461232781410217, 0.021795086562633514, 0.023419681936502457, 0.03386644273996353, 0.028580598533153534, 0.03440327197313309, 0.033156976103782654, 0.026710400357842445, 0.04213565215468407, 0.019021350890398026, 0.032785814255476, 0.02683998830616474, 0.03304455429315567, 0.02330264076590538, 0.03390035033226013, 0.03944924846291542, 0.021334253251552582, 0.042546432465314865, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10210391879081726, 0.048305340111255646, 0.027957914397120476, 0.028134863823652267, 0.04368893802165985, 0.03765960410237312, 0.04012182354927063, 0.03589581325650215, 0.024491794407367706, 0.019768042489886284, 0.03467409312725067, 0.015431461855769157, 0.032616548240184784, 0.03414339944720268, 0.027461368590593338, 0.020323220640420914, 0.03445841744542122, 0.015827562659978867, 0.030705846846103668, 0.020360933616757393, 0.03558044880628586, 0.034790296107530594, 0.03518148139119148, 0.028572604060173035, 0.04333385452628136, 0.03896389529109001, 0.017587700858712196, 0.04553375393152237, 0.04632510989904404, 0.0, 0.0, 0.0, 0.0], [0.1184174120426178, 0.026689782738685608, 0.02854764647781849, 0.03516455739736557, 0.03519093617796898, 0.030704878270626068, 0.042546890676021576, 0.023103507235646248, 0.025463124737143517, 0.029678549617528915, 0.028395220637321472, 0.019178256392478943, 0.02383357845246792, 0.027640020474791527, 0.03381752222776413, 0.0396401509642601, 0.027574418112635612, 0.021856164559721947, 0.03966765105724335, 0.03941834345459938, 0.02711562067270279, 0.030498063191771507, 0.034932829439640045, 0.0323205292224884, 0.027840541675686836, 0.03877578675746918, 0.021124795079231262, 0.03861142694950104, 0.032771822065114975, 0.019479982554912567, 0.0, 0.0, 0.0], [0.05433792620897293, 0.019247835502028465, 0.011686461977660656, 0.014118609949946404, 0.01390978042036295, 0.05736807361245155, 0.0345463827252388, 0.046650584787130356, 0.013026447966694832, 0.013271295465528965, 0.05952775105834007, 0.009063147008419037, 0.017364196479320526, 0.061355650424957275, 0.01263465266674757, 0.015145537443459034, 0.06425532698631287, 0.046270616352558136, 0.01955636776983738, 0.012543462216854095, 0.06913558393716812, 0.034823667258024216, 0.030005954205989838, 0.012261823751032352, 0.07985591888427734, 0.018996428698301315, 0.010722929611802101, 0.0865454226732254, 0.017174094915390015, 0.022144164890050888, 0.022453978657722473, 0.0, 0.0], [0.05849646031856537, 0.03437737002968788, 0.013603095896542072, 0.02417043223977089, 0.01900586113333702, 0.0654640942811966, 0.030622966587543488, 0.03573206812143326, 0.01885945349931717, 0.012241484597325325, 0.06282190978527069, 0.011835752986371517, 0.009552359580993652, 0.06431794911623001, 0.02158958464860916, 0.010912974365055561, 0.0666591003537178, 0.02399745211005211, 0.026596758514642715, 0.006153556052595377, 0.06984329968690872, 0.03723869100213051, 0.030046707019209862, 0.021686814725399017, 0.043640609830617905, 0.021209606900811195, 0.013490457087755203, 0.04632722586393356, 0.020434511825442314, 0.013362264260649681, 0.018568990752100945, 0.04714024439454079, 0.0], [0.1352868676185608, 0.02973337098956108, 0.02611132338643074, 0.03753575682640076, 0.038881585001945496, 0.0331667996942997, 0.04824390634894371, 0.03046831116080284, 0.026038208976387978, 0.024123696610331535, 0.027892732992768288, 0.02270994521677494, 0.024185286834836006, 0.02611454948782921, 0.02320966310799122, 0.025175290182232857, 0.024614792317152023, 0.02112315408885479, 0.02968505024909973, 0.01816747523844242, 0.023064401000738144, 0.02059807814657688, 0.034411393105983734, 0.021083395928144455, 0.02253883145749569, 0.0322815477848053, 0.013987251557409763, 0.02414492331445217, 0.02700231224298477, 0.012929446063935757, 0.03616539016366005, 0.02684023790061474, 0.03248502314090729]]]}\n",
        "    )\n",
        "    </script>"
       ],
       "text/plain": [
-       "<circuitsvis.utils.render.RenderedHTML at 0x7f208db438d0>"
+       "<circuitsvis.utils.render.RenderedHTML at 0x7f282c95c950>"
       ]
      },
      "execution_count": 13,
@@ -552,10 +552,10 @@
    "execution_count": 14,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:41:50.912908Z",
-     "iopub.status.busy": "2024-09-05T17:41:50.912591Z",
-     "iopub.status.idle": "2024-09-05T17:41:51.172080Z",
-     "shell.execute_reply": "2024-09-05T17:41:51.171439Z"
+     "iopub.execute_input": "2024-09-10T17:15:29.388142Z",
+     "iopub.status.busy": "2024-09-10T17:15:29.387782Z",
+     "iopub.status.idle": "2024-09-10T17:15:29.656354Z",
+     "shell.execute_reply": "2024-09-10T17:15:29.655751Z"
     }
    },
    "outputs": [
@@ -638,10 +638,10 @@
    "execution_count": 15,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:41:51.174688Z",
-     "iopub.status.busy": "2024-09-05T17:41:51.174272Z",
-     "iopub.status.idle": "2024-09-05T17:41:51.388302Z",
-     "shell.execute_reply": "2024-09-05T17:41:51.387650Z"
+     "iopub.execute_input": "2024-09-10T17:15:29.658923Z",
+     "iopub.status.busy": "2024-09-10T17:15:29.658568Z",
+     "iopub.status.idle": "2024-09-10T17:15:29.869347Z",
+     "shell.execute_reply": "2024-09-10T17:15:29.868751Z"
     }
    },
    "outputs": [
@@ -694,17 +694,17 @@
    "execution_count": 16,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:41:51.390505Z",
-     "iopub.status.busy": "2024-09-05T17:41:51.390296Z",
-     "iopub.status.idle": "2024-09-05T17:42:11.721994Z",
-     "shell.execute_reply": "2024-09-05T17:42:11.721377Z"
+     "iopub.execute_input": "2024-09-10T17:15:29.871644Z",
+     "iopub.status.busy": "2024-09-10T17:15:29.871447Z",
+     "iopub.status.idle": "2024-09-10T17:15:50.138355Z",
+     "shell.execute_reply": "2024-09-10T17:15:50.137730Z"
     }
    },
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "3829b426693a411babc613fc1d68b1c8",
+       "model_id": "8de918cf34204469a9a6abae1699ad36",
        "version_major": 2,
        "version_minor": 0
       },
@@ -761,10 +761,10 @@
    "execution_count": 17,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:42:11.724472Z",
-     "iopub.status.busy": "2024-09-05T17:42:11.724028Z",
-     "iopub.status.idle": "2024-09-05T17:42:11.878609Z",
-     "shell.execute_reply": "2024-09-05T17:42:11.877989Z"
+     "iopub.execute_input": "2024-09-10T17:15:50.140857Z",
+     "iopub.status.busy": "2024-09-10T17:15:50.140469Z",
+     "iopub.status.idle": "2024-09-10T17:15:50.292966Z",
+     "shell.execute_reply": "2024-09-10T17:15:50.292392Z"
     }
    },
    "outputs": [
@@ -775,9 +775,9 @@
        "<head><meta charset=\"utf-8\" /></head>\n",
        "<body>\n",
        "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
-       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"ac2799ee-2c40-4026-87e5-7bc5d779bc16\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"ac2799ee-2c40-4026-87e5-7bc5d779bc16\")) {                    Plotly.newPlot(                        \"ac2799ee-2c40-4026-87e5-7bc5d779bc16\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"x\":[\"\\u003c|endoftext|\\u003e_0\",\"After_1\",\" John_2\",\" and_3\",\" Mary_4\",\" went_5\",\" to_6\",\" the_7\",\" store_8\",\",_9\",\" Mary_10\",\" gave_11\",\" a_12\",\" bottle_13\",\" of_14\",\" milk_15\",\" to_16\"],\"z\":[[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9981481432914734,0.001600604155100882,0.0001503795647295192,-0.0003718698862940073,-2.257052983622998e-05,-0.000628303736448288,-0.000515179184731096],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9980564713478088,0.002283838577568531,0.00018205988453701138,-0.0005045737489126623,-0.0002681270125322044,-5.153151141712442e-05,-0.0012816254748031497],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9967373609542847,0.004081866703927517,0.0009732521721161902,4.378139055916108e-05,-0.00015962531324476004,-0.00033638247987255454,-0.0019441929180175066],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9905908107757568,0.019987020641565323,0.0018955166451632977,0.0010141782695427537,-6.75756236887537e-05,0.000911251176148653,-0.0019012272823601961],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9616515636444092,0.08534706383943558,0.0052040028385818005,0.003052052343264222,0.00019688029715325683,0.0011058200616389513,-0.002284654416143894],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.963099479675293,0.0843719094991684,0.00412211287766695,0.0007176341023296118,0.00010319898865418509,0.0010020771296694875,-0.004215250723063946],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9359182119369507,0.11111730337142944,0.007705116644501686,0.0003754050412680954,0.0003647996054496616,0.0013266304740682244,0.01874428242444992],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.7701544761657715,0.03741976246237755,0.0020680588204413652,-8.321183850057423e-05,0.00013447141100186855,0.0017246060306206346,0.44990694522857666],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0965060144662857,0.025925789028406143,0.0019711144268512726,0.00032904025283642113,0.00042367333662696183,0.00188477523624897,0.8994728922843933],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.023322155699133873,0.018537340685725212,0.001587143400683999,0.000526600400917232,0.0002531706413719803,0.0008727724780328572,0.9612765908241272],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.008559261448681355,0.006339463870972395,0.0005807152483612299,-0.000342636980349198,0.00010958944039884955,0.0006486988277174532,0.9495824575424194]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Position: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Position\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Normalized Logit Difference After Patching Residual Stream on the IOI Task\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"8ba8ca06-6030-4e88-87b2-a1ff0dfb3302\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"8ba8ca06-6030-4e88-87b2-a1ff0dfb3302\")) {                    Plotly.newPlot(                        \"8ba8ca06-6030-4e88-87b2-a1ff0dfb3302\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"x\":[\"\\u003c|endoftext|\\u003e_0\",\"After_1\",\" John_2\",\" and_3\",\" Mary_4\",\" went_5\",\" to_6\",\" the_7\",\" store_8\",\",_9\",\" Mary_10\",\" gave_11\",\" a_12\",\" bottle_13\",\" of_14\",\" milk_15\",\" to_16\"],\"z\":[[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9981481432914734,0.001600604155100882,0.0001503795647295192,-0.0003718698862940073,-2.257052983622998e-05,-0.000628303736448288,-0.000515179184731096],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9980564713478088,0.002283838577568531,0.00018205988453701138,-0.0005045737489126623,-0.0002681270125322044,-5.153151141712442e-05,-0.0012816254748031497],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9967373609542847,0.004081866703927517,0.0009732521721161902,4.378139055916108e-05,-0.00015962531324476004,-0.00033638247987255454,-0.0019441929180175066],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9905908107757568,0.019987020641565323,0.0018955166451632977,0.0010141782695427537,-6.75756236887537e-05,0.000911251176148653,-0.0019012272823601961],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9616515636444092,0.08534706383943558,0.0052040028385818005,0.003052052343264222,0.00019688029715325683,0.0011058200616389513,-0.002284654416143894],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.963099479675293,0.0843719094991684,0.00412211287766695,0.0007176341023296118,0.00010319898865418509,0.0010020771296694875,-0.004215250723063946],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9359182119369507,0.11111730337142944,0.007705116644501686,0.0003754050412680954,0.0003647996054496616,0.0013266304740682244,0.01874428242444992],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.7701544761657715,0.03741976246237755,0.0020680588204413652,-8.321183850057423e-05,0.00013447141100186855,0.0017246060306206346,0.44990694522857666],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0965060144662857,0.025925789028406143,0.0019711144268512726,0.00032904025283642113,0.00042367333662696183,0.00188477523624897,0.8994728922843933],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.023322155699133873,0.018537340685725212,0.001587143400683999,0.000526600400917232,0.0002531706413719803,0.0008727724780328572,0.9612765908241272],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.008559261448681355,0.006339463870972395,0.0005807152483612299,-0.000342636980349198,0.00010958944039884955,0.0006486988277174532,0.9495824575424194]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Position: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Position\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Normalized Logit Difference After Patching Residual Stream on the IOI Task\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('ac2799ee-2c40-4026-87e5-7bc5d779bc16');\n",
+       "var gd = document.getElementById('8ba8ca06-6030-4e88-87b2-a1ff0dfb3302');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -854,10 +854,10 @@
    "execution_count": 18,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:42:11.881029Z",
-     "iopub.status.busy": "2024-09-05T17:42:11.880576Z",
-     "iopub.status.idle": "2024-09-05T17:42:13.694216Z",
-     "shell.execute_reply": "2024-09-05T17:42:13.693623Z"
+     "iopub.execute_input": "2024-09-10T17:15:50.295410Z",
+     "iopub.status.busy": "2024-09-10T17:15:50.295059Z",
+     "iopub.status.idle": "2024-09-10T17:15:52.104339Z",
+     "shell.execute_reply": "2024-09-10T17:15:52.103783Z"
     }
    },
    "outputs": [
@@ -868,9 +868,9 @@
        "<head><meta charset=\"utf-8\" /></head>\n",
        "<body>\n",
        "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
-       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"b5559adb-b6df-4664-85da-aad13d04f74a\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"b5559adb-b6df-4664-85da-aad13d04f74a\")) {                    Plotly.newPlot(                        \"b5559adb-b6df-4664-85da-aad13d04f74a\",                        [{\"hovertemplate\":\"variable=0\\u003cbr\\u003eindex=%{x}\\u003cbr\\u003evalue=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"0\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"0\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98],\"xaxis\":\"x\",\"y\":[11.219001770019531,12.648847579956055,16.002866744995117,12.655248641967773,13.598546981811523,12.648005485534668,13.230877876281738,11.950650215148926,12.584773063659668,13.332685470581055,11.372512817382812,13.18463134765625,12.104430198669434,12.645095825195312,13.332456588745117,12.490678787231445,13.130325317382812,12.238739967346191,13.111671447753906,13.072357177734375,11.822067260742188,12.05682373046875,12.79888916015625,12.196507453918457,12.406768798828125,11.383073806762695,12.965502738952637,11.637373924255371,10.67663860321045,11.894469261169434,13.117510795593262,11.437427520751953,11.200849533081055,11.569291114807129,10.730561256408691,12.751376152038574,11.765420913696289,12.07258415222168,11.446783065795898,11.796368598937988,11.440250396728516,11.975069046020508,10.75130844116211,10.960607528686523,11.075111389160156,10.82270622253418,11.122757911682129,11.248815536499023,12.138458251953125,10.384830474853516,2.522568702697754,1.1026382446289062,0.7128584384918213,0.4818822741508484,0.31920477747917175,0.15357092022895813,0.13818064332008362,0.16000370681285858,0.23586773872375488,0.06869865953922272,0.21874837577342987,0.11845606565475464,0.17535646259784698,0.0625605583190918,0.027201101183891296,0.07039335370063782,0.03363990783691406,0.047253381460905075,0.06748445332050323,0.013322415761649609,0.027032632380723953,0.21386680006980896,0.041274793446063995,0.14630667865276337,0.01946776732802391,0.11438588798046112,0.04898519441485405,0.027135217562317848,0.05644571781158447,0.11561702191829681,0.1028728261590004,0.049143046140670776,0.018940147012472153,0.028789499774575233,0.04571650177240372,0.009683424606919289,0.049352310597896576,0.05664796754717827,0.18444031476974487,0.06125975772738457,0.25522834062576294,0.4084078371524811,0.0669250562787056,0.02292284369468689,0.15346066653728485,0.07391653954982758,0.06822908669710159,0.0359528474509716,0.07923130691051483],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"index\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"value\"}},\"legend\":{\"title\":{\"text\":\"variable\"},\"tracegroupgap\":0},\"title\":{\"text\":\"Loss by position on random repeated tokens\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"bf609849-a683-4501-98c0-5610402069af\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"bf609849-a683-4501-98c0-5610402069af\")) {                    Plotly.newPlot(                        \"bf609849-a683-4501-98c0-5610402069af\",                        [{\"hovertemplate\":\"variable=0\\u003cbr\\u003eindex=%{x}\\u003cbr\\u003evalue=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"0\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"0\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98],\"xaxis\":\"x\",\"y\":[11.916303634643555,14.13970947265625,13.97419548034668,13.20025634765625,12.98205280303955,12.23963737487793,13.160758972167969,12.960711479187012,12.692301750183105,11.697540283203125,13.213212966918945,12.825567245483398,11.889198303222656,10.807716369628906,12.493316650390625,12.748961448669434,11.875480651855469,11.771799087524414,12.129152297973633,12.251404762268066,10.839369773864746,12.363996505737305,11.972248077392578,12.573539733886719,12.151954650878906,11.586162567138672,12.556215286254883,11.899218559265137,11.944257736206055,11.11519718170166,11.516045570373535,11.446038246154785,12.16328239440918,11.093135833740234,11.868314743041992,10.437663078308105,12.832025527954102,11.429876327514648,10.88789176940918,10.855405807495117,11.266473770141602,11.21790885925293,11.649023056030273,11.6561861038208,12.299074172973633,11.253116607666016,11.278722763061523,11.215995788574219,10.433588027954102,10.897237777709961,3.044490098953247,1.069545030593872,1.016643762588501,0.4091116786003113,0.432170569896698,0.24384959042072296,0.7159929871559143,0.2306588888168335,0.14308522641658783,0.2239038646221161,0.06011886149644852,0.16535978019237518,0.2879369556903839,0.040467940270900726,0.2961328625679016,0.14453032612800598,0.10808869451284409,0.08187573403120041,0.09832978248596191,0.052005164325237274,0.04744909703731537,0.12793658673763275,0.08489024639129639,0.03772114962339401,0.04858517646789551,0.08634428679943085,0.10197576135396957,0.021571611985564232,0.09415853023529053,0.06979478150606155,0.16457553207874298,0.04652702435851097,0.009943359531462193,0.039533548057079315,0.10102002322673798,0.022349249571561813,0.18824195861816406,0.03961147740483284,0.024878712370991707,0.0205758698284626,0.05781855434179306,0.014255058951675892,0.0838586762547493,0.024006284773349762,0.9298907518386841,0.0951559990644455,0.03629877790808678,0.07244053483009338,0.044225141406059265],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"index\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"value\"}},\"legend\":{\"title\":{\"text\":\"variable\"},\"tracegroupgap\":0},\"title\":{\"text\":\"Loss by position on random repeated tokens\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('b5559adb-b6df-4664-85da-aad13d04f74a');\n",
+       "var gd = document.getElementById('bf609849-a683-4501-98c0-5610402069af');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -936,10 +936,10 @@
    "execution_count": 19,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:42:13.696398Z",
-     "iopub.status.busy": "2024-09-05T17:42:13.696218Z",
-     "iopub.status.idle": "2024-09-05T17:42:14.900462Z",
-     "shell.execute_reply": "2024-09-05T17:42:14.899922Z"
+     "iopub.execute_input": "2024-09-10T17:15:52.106634Z",
+     "iopub.status.busy": "2024-09-10T17:15:52.106281Z",
+     "iopub.status.idle": "2024-09-10T17:15:53.318758Z",
+     "shell.execute_reply": "2024-09-10T17:15:53.318170Z"
     }
    },
    "outputs": [
@@ -950,9 +950,9 @@
        "<head><meta charset=\"utf-8\" /></head>\n",
        "<body>\n",
        "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
-       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"abbc7990-fb5b-4a65-ba89-78a2073c73df\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"abbc7990-fb5b-4a65-ba89-78a2073c73df\")) {                    Plotly.newPlot(                        \"abbc7990-fb5b-4a65-ba89-78a2073c73df\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.010287518613040447,7.07331855664961e-05,0.01022765040397644,9.369629765387799e-07,0.0001593400229467079,6.424106686608866e-05,0.009279504418373108,0.0007229120819829404,0.00916075985878706,0.009111087769269943,0.006602954585105181,0.0152360824868083],[0.0005463710404001176,0.00035243277670815587,0.0021286087576299906,0.014092736877501011,0.005032995715737343,0.010190492495894432,0.015489863231778145,0.012780369259417057,0.012919844128191471,0.01602115109562874,0.0064891609363257885,0.0004646908782888204],[0.004244121257215738,0.018242545425891876,0.0023879925720393658,0.0012670502765104175,0.013606147840619087,0.0018331799656152725,0.0033160552848130465,0.00845535658299923,0.0038152106571942568,0.0014914380153641105,0.0006890674121677876,0.01019816193729639],[0.01416884083300829,0.006844150368124247,0.002567051211372018,0.01341474149376154,0.021600881591439247,0.01306712906807661,0.001567754428833723,0.0013694057706743479,0.0052207414992153645,0.01306647714227438,0.009408000856637955,0.007939903065562248],[0.015263119712471962,0.013007179833948612,0.014736704528331757,0.008307430893182755,0.018053121864795685,0.01217469573020935,0.008762415498495102,0.002206446137279272,0.015867728739976883,0.013769634068012238,0.018624909222126007,9.13730691110004e-09],[0.47185665369033813,0.9086124897003174,0.01296070497483015,0.006677804049104452,0.009637922048568726,0.9290925860404968,0.01033448614180088,0.018294984474778175,0.02779943309724331,0.02766416408121586,0.017583714798092842,0.017298221588134766],[0.007537637371569872,0.017424091696739197,0.018900733441114426,0.01431836187839508,0.022236179560422897,0.011647127568721771,0.03073083981871605,0.011905021034181118,0.008411239832639694,0.9175557494163513,0.03838243708014488,0.013622810132801533],[0.01000579446554184,0.17460721731185913,0.8657189011573792,0.017993038520216942,0.01724315620958805,0.018139807507395744,0.04568207263946533,0.09614809602499008,0.01781415194272995,0.018681053072214127,0.9199892282485962,0.05816270038485527],[0.016772812232375145,0.4192027151584625,0.016147194430232048,0.05109066516160965,0.016613438725471497,0.012248117476701736,0.1683734506368637,0.011353541165590286,0.03254345804452896,0.031994905322790146,0.07066226005554199,0.022813519462943077],[0.2658080458641052,0.18101611733436584,0.1085599958896637,0.01112457923591137,0.09504234045743942,0.027309555560350418,0.47915005683898926,0.03030301257967949,0.05258383974432945,0.4882603585720062,0.015520771034061909,0.04620152711868286],[0.36585161089897156,0.5181708335876465,0.03953804820775986,0.1504061222076416,0.05925014615058899,0.014007902704179287,0.31749680638313293,0.4777964651584625,0.04831257089972496,0.014951827004551888,0.1641472578048706,0.2731083333492279],[0.017277132719755173,0.05256804823875427,0.037828266620635986,0.010270046070218086,0.03663536533713341,0.0975709855556488,0.05082761123776436,0.06714604049921036,0.007663114927709103,0.3211846351623535,0.3871070444583893,0.02293839119374752]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Induction Score by Head\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"d4826cd4-d822-446a-9486-7f010e673bbf\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"d4826cd4-d822-446a-9486-7f010e673bbf\")) {                    Plotly.newPlot(                        \"d4826cd4-d822-446a-9486-7f010e673bbf\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.010405071079730988,8.505234291078523e-05,0.010846362449228764,1.542941731713654e-06,0.00015207800606731325,0.00016587963909842074,0.00962826982140541,0.0005811225273646414,0.00920275691896677,0.009271877817809582,0.006795060820877552,0.015113143250346184],[0.0007497637998312712,0.0003636373148765415,0.002115159062668681,0.014394569210708141,0.005006241612136364,0.010394004173576832,0.015367378480732441,0.01268749125301838,0.012691996991634369,0.016459191218018532,0.006484598852694035,0.0005559310666285455],[0.004768972750753164,0.01647953875362873,0.0027482376899570227,0.0013796724379062653,0.012924875132739544,0.0018674784805625677,0.003705371869727969,0.008166997693479061,0.004057642072439194,0.0012803683057427406,0.000662333972286433,0.010387937538325787],[0.015679387375712395,0.007625625003129244,0.002122837584465742,0.013503310270607471,0.02086609974503517,0.01211901567876339,0.001721711247228086,0.0009715322521515191,0.00451809773221612,0.01239688228815794,0.008685757406055927,0.006643661297857761],[0.016919558867812157,0.01504963357001543,0.014352007769048214,0.008405109867453575,0.01849125511944294,0.01381102204322815,0.009090271778404713,0.001800192752853036,0.013694416731595993,0.014873025007545948,0.018977493047714233,9.226908126436228e-09],[0.4363614022731781,0.9093427658081055,0.014157562516629696,0.007539841346442699,0.011399139650166035,0.9306832551956177,0.010265758261084557,0.017486363649368286,0.024969371035695076,0.027117766439914703,0.018408896401524544,0.01754666306078434],[0.009883408434689045,0.017079785466194153,0.0185578390955925,0.015524016693234444,0.020320380106568336,0.012695600278675556,0.030558481812477112,0.012052788399159908,0.01120459008961916,0.916867733001709,0.032972753047943115,0.01364062912762165],[0.008361412212252617,0.171825110912323,0.8461205363273621,0.018025416880846024,0.018446065485477448,0.01776888407766819,0.04303755983710289,0.08179712295532227,0.015604961663484573,0.018940657377243042,0.9122208952903748,0.05450733006000519],[0.014317817986011505,0.3731990456581116,0.014678257517516613,0.04617224633693695,0.017426906153559685,0.01191379688680172,0.1442830115556717,0.013355129398405552,0.029831815510988235,0.03103570081293583,0.06533283740282059,0.021598415449261665],[0.24887625873088837,0.1976928859949112,0.1104423925280571,0.012795306742191315,0.09626349806785583,0.02669103629887104,0.44758379459381104,0.028052648529410362,0.05248158425092697,0.4636542797088623,0.01666952669620514,0.041144587099552155],[0.3466327488422394,0.5092848539352417,0.03988264873623848,0.1478113979101181,0.06403890997171402,0.01558593474328518,0.29378482699394226,0.4561544358730316,0.052708882838487625,0.016264332458376884,0.1581176519393921,0.2725672125816345],[0.017380494624376297,0.05578174069523811,0.03640053793787956,0.009598813019692898,0.0362129844725132,0.10631337016820908,0.054979126900434494,0.06522571295499802,0.007772110868245363,0.3159135580062866,0.40592387318611145,0.021894147619605064]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Induction Score by Head\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('abbc7990-fb5b-4a65-ba89-78a2073c73df');\n",
+       "var gd = document.getElementById('d4826cd4-d822-446a-9486-7f010e673bbf');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -1027,28 +1027,28 @@
    "execution_count": 20,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:42:14.902531Z",
-     "iopub.status.busy": "2024-09-05T17:42:14.902332Z",
-     "iopub.status.idle": "2024-09-05T17:42:15.116306Z",
-     "shell.execute_reply": "2024-09-05T17:42:15.115643Z"
+     "iopub.execute_input": "2024-09-10T17:15:53.321102Z",
+     "iopub.status.busy": "2024-09-10T17:15:53.320758Z",
+     "iopub.status.idle": "2024-09-10T17:15:53.545304Z",
+     "shell.execute_reply": "2024-09-10T17:15:53.544639Z"
     }
    },
    "outputs": [
     {
      "data": {
       "text/html": [
-       "<div id=\"circuits-vis-34e21d87-69ed\" style=\"margin: 15px 0;\"/>\n",
+       "<div id=\"circuits-vis-0896baec-5f5f\" style=\"margin: 15px 0;\"/>\n",
        "    <script crossorigin type=\"module\">\n",
        "    import { render, AttentionPatterns } from \"https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js\";\n",
        "    render(\n",
-       "      \"circuits-vis-34e21d87-69ed\",\n",
+       "      \"circuits-vis-0896baec-5f5f\",\n",
        "      AttentionPatterns,\n",
        "      {\"tokens\": [\"use\", \" advice\", \" Social\", \"\\u00f6\", \"\\u00b7\", \" fought\", \" Le\", \" allegedly\", \" NO\", \"alth\", \"car\", \" prepared\", \"new\", \"rant\", \"roll\", \" hours\", \" published\", \"66\", \"ension\", \" 44\", \"use\", \" advice\", \" Social\", \"\\u00f6\", \"\\u00b7\", \" fought\", \" Le\", \" allegedly\", \" NO\", \"alth\", \"car\", \" prepared\", \"new\", \"rant\", \"roll\", \" hours\", \" published\", \"66\", \"ension\", \" 44\"], \"attention\": [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9737270474433899, 0.02627299167215824, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9820428490638733, 0.017020218074321747, 0.0009368443279527128, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9895542860031128, 0.008665801025927067, 0.00041197543032467365, 0.0013679895782843232, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8543054461479187, 0.07801821082830429, 0.0008415375486947596, 0.0001359905581921339, 0.06669880449771881, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.937433660030365, 0.03300206735730171, 0.0015577399171888828, 2.535264457037556e-06, 0.0010925547685474157, 0.026911458000540733, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9769211411476135, 0.0038436795584857464, 2.234023304481525e-05, 3.5218730772612616e-05, 0.00518348254263401, 0.012176254764199257, 0.0018179015023633838, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9473504424095154, 0.013174979016184807, 0.0013492192374542356, 1.1802491826529149e-05, 0.0009449502103962004, 0.01131904125213623, 0.018021006137132645, 0.00782855786383152, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9847128391265869, 0.0010781444143503904, 0.0021734496112912893, 5.4822653510200325e-06, 0.0004914247547276318, 0.0013570826267823577, 0.00010185814608121291, 0.0002853855839930475, 0.009794436395168304, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9915198683738708, 0.004483341239392757, 0.00012727164721582085, 0.00016702155699022114, 0.0016301742289215326, 0.0011521608103066683, 0.00032312856637872756, 0.00012646272080019116, 0.00039313736488111317, 7.735046528978273e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8908806443214417, 0.024311941117048264, 1.7341310012852773e-05, 4.157719376962632e-05, 0.0008967609610408545, 0.07334917038679123, 0.0009482790483161807, 0.004280843771994114, 0.00516867870464921, 7.830337381165009e-06, 9.693232277641073e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8170808553695679, 0.1351710557937622, 0.011989916674792767, 1.1421708222769666e-05, 0.0003511959221214056, 0.009450665675103664, 0.01946333795785904, 0.0006557459710165858, 0.0005760999629274011, 2.9927030482213013e-05, 1.6589872757322155e-05, 0.00520310876891017, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9082695841789246, 0.006068154238164425, 0.013871855102479458, 0.0008237077272497118, 0.01190832071006298, 0.015542060136795044, 0.008354832418262959, 0.002078164601698518, 0.0013173240004107356, 0.0021398039534687996, 0.0039441632106900215, 0.0012376613449305296, 0.024444298818707466, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9479591250419617, 0.0021026728209108114, 0.011938490904867649, 0.00012338497617747635, 3.5374921480979538e-06, 0.00014498813834507018, 0.0005875456845387816, 2.5534465748933144e-05, 0.0013608969748020172, 0.0003395687963347882, 0.010076268576085567, 0.015790557488799095, 0.006346334703266621, 0.0032010283321142197, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9393549561500549, 0.006392320618033409, 0.0018427377799525857, 6.1162154452176765e-06, 0.00033587400685064495, 0.002051525516435504, 0.003801509505137801, 0.0012357983505353332, 0.00021948102221358567, 0.0003869360953103751, 5.012243491364643e-05, 0.008153212256729603, 0.026924701407551765, 0.002937993500381708, 0.0063067772425711155, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9339622259140015, 0.0017828107811510563, 0.005864636041224003, 0.0001995021157199517, 7.227134483400732e-05, 0.0014535371446982026, 0.0025924311485141516, 0.00048593789688311517, 0.002229832811281085, 0.00015120484749786556, 0.012292915023863316, 0.005057850386947393, 0.01236859429627657, 0.0039444840513169765, 0.006275098770856857, 0.011266663670539856, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8931334018707275, 0.0015468214405700564, 0.013001682236790657, 7.966378689161502e-06, 5.864375998498872e-05, 0.0008863414404913783, 0.00320208678022027, 3.214761454728432e-05, 0.00018022459698840976, 1.1455733329057693e-05, 7.600126264151186e-05, 0.00042027299059554935, 0.0016126197297126055, 0.028539275750517845, 0.01053555216640234, 0.025432037189602852, 0.021323570981621742, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9847024083137512, 0.000458244961919263, 0.00017220497829839587, 6.160992711556901e-07, 4.782819814863615e-06, 0.0005806140834465623, 0.0004461870121303946, 0.0004120134108234197, 0.0013038699980825186, 0.0003176067257300019, 6.994167779339477e-05, 0.0013941116631031036, 5.583090751315467e-05, 0.0009110373794101179, 0.00019558057829272002, 0.0003960288013331592, 0.0011691722320392728, 0.007409730460494757, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8922597765922546, 0.01028366107493639, 0.007569293025881052, 0.015225829556584358, 0.0006035296246409416, 0.0014377726474776864, 0.018397418782114983, 0.00018186635861638933, 0.002113566268235445, 3.803668369073421e-05, 0.009962561540305614, 0.003998206928372383, 0.0012666863622143865, 0.0021862699650228024, 0.0032670684158802032, 0.0015871950890868902, 0.01913381926715374, 0.008779395371675491, 0.0017080748220905662, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5637838244438171, 2.204169322794769e-05, 0.0003808305482380092, 1.393800062032824e-07, 4.306913226059805e-08, 0.00012883225281257182, 5.202722968533635e-05, 4.098194494872587e-06, 0.00043821518192999065, 1.0102487067342736e-05, 2.0490055248956196e-05, 0.00021747536084149033, 2.5250110411434434e-05, 2.129377389792353e-05, 0.0022071911953389645, 5.892770423088223e-05, 0.002418374642729759, 0.003277554176747799, 0.42604929208755493, 0.0008841048693284392, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14635005593299866, 0.45109108090400696, 0.027205264195799828, 0.0030082776211202145, 0.0007913765148259699, 0.0800931379199028, 0.005927647929638624, 0.0006846337346360087, 0.0021268464624881744, 0.0027747468557208776, 0.00023907337163109332, 0.002550546545535326, 0.0054934462532401085, 0.01583220437169075, 0.0003450022195465863, 0.0005726688541471958, 0.002175167202949524, 0.039043255150318146, 0.16982686519622803, 0.04120763763785362, 0.0026610149070620537, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15883015096187592, 0.09413880854845047, 0.6926899552345276, 4.776390778715722e-05, 8.085349691100419e-06, 0.00935556460171938, 0.0008445517742075026, 2.443764515192015e-06, 0.00013773982936982065, 1.1189692941115936e-06, 4.677353445003973e-06, 0.0003472122480161488, 0.002631494775414467, 0.0004504133830778301, 0.006463215220719576, 0.0005723336944356561, 0.0012668337440118194, 0.006402322091162205, 0.0018092951504513621, 0.006555411033332348, 0.00037914939457550645, 0.017061391845345497, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05072391405701637, 0.0043561081402003765, 0.00013167233555577695, 0.9396469593048096, 0.0005500844563357532, 0.002771171974018216, 1.4556275345967151e-05, 5.0173721319879405e-06, 1.5498681023018435e-05, 7.020712899930004e-08, 8.694933967490215e-06, 3.654160536825657e-05, 3.607992539400584e-06, 2.5941648345906287e-05, 7.590860604977934e-06, 7.10092763256398e-07, 4.6298235247377306e-05, 7.143522088881582e-05, 0.00012089123629266396, 0.0005609994404949248, 1.3380947166297119e-05, 0.0007342157769016922, 0.0001547116116853431, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.044663090258836746, 0.0028326627798378468, 7.648707105545327e-05, 0.00015513667312916368, 0.7502228021621704, 0.19195663928985596, 9.640512871555984e-05, 0.00016210127796512097, 0.00012769461318384856, 1.1226569768041372e-05, 8.733285540074576e-06, 0.0002813320606946945, 5.2078037697356194e-05, 0.008386553265154362, 4.340289706306066e-06, 6.482724711531773e-05, 3.802955325227231e-05, 7.603636913700029e-05, 0.00012636324390769005, 9.227949340129271e-05, 4.0301659964825376e-07, 0.00011281677143415436, 2.522385329939425e-06, 0.0004493836604524404, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.013606224209070206, 0.006971816532313824, 3.1886105716694146e-05, 1.8455255030858098e-06, 0.002301084576174617, 0.9711790084838867, 0.0003632458101492375, 8.45977701828815e-05, 0.0001061156508512795, 4.5056845010549296e-07, 2.987579534874385e-07, 0.0008592646918259561, 8.844918920658529e-05, 0.00034812072408385575, 9.285179203288862e-07, 3.160578853567131e-05, 1.2802144738088828e-05, 5.8033951063407585e-05, 0.00010517514601815492, 6.438296259148046e-05, 1.8867377775677596e-06, 0.0009238758939318359, 4.68101461592596e-06, 8.99309179658303e-06, 0.0028452184051275253, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.01961127668619156, 0.00438615120947361, 6.198724440764636e-05, 4.88576041846045e-08, 4.252470171195455e-05, 0.0036122461315244436, 0.9598402380943298, 0.005622219294309616, 0.0023445591796189547, 5.173985186956997e-07, 1.9621675164671615e-06, 0.00165482924785465, 0.0005915391957387328, 0.001169584458693862, 5.784227596450364e-06, 8.119036647258326e-05, 4.500302384258248e-05, 0.00018497802375350147, 5.086949386168271e-05, 0.00011132832878502086, 6.848460998298833e-06, 3.8539848901564255e-05, 4.853054633713327e-06, 1.276350758416811e-07, 3.953097802877892e-06, 0.0005269531393423676, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.008729088120162487, 0.0001781070022843778, 2.8806286422877747e-07, 4.138263136610476e-07, 6.864999886602163e-05, 0.0009331137989647686, 0.0001488832349423319, 0.984452486038208, 0.004786369390785694, 0.0001132894612965174, 7.255491141222592e-07, 7.424689101753756e-05, 1.4996417121437844e-05, 0.00019790598889812827, 2.995051318066544e-07, 4.872014415013837e-06, 5.296082690620096e-06, 4.536818778433371e-06, 0.00011268883827142417, 3.4170998333138414e-06, 8.272462764580268e-06, 1.23798881759285e-05, 3.665505587946427e-08, 9.0251188566981e-07, 1.6053050785558298e-05, 0.00011404338874854147, 1.8653627193998545e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04082155600190163, 0.003618426388129592, 4.821046968572773e-05, 1.9615674773376668e-07, 5.3802777983946726e-05, 0.0021802401170134544, 0.0033272523432970047, 0.001878200564533472, 0.9307873249053955, 0.00481536565348506, 1.443025394110009e-05, 0.0028287956956773996, 0.00010464123624842614, 0.0029309086967259645, 0.0009514776756986976, 6.522179319290444e-05, 0.00029541170806623995, 0.00012317505024839193, 0.0015512284589931369, 0.0005327722756192088, 0.0002208336372859776, 0.00044327162322588265, 1.9391492969589308e-05, 5.825700100103859e-07, 1.6632819097139873e-05, 0.0004773235123138875, 0.0011258182348683476, 0.0007676867535337806, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.13800986111164093, 0.000338585494318977, 0.00015972793335095048, 3.422598027214008e-08, 1.2696453268290497e-05, 0.00024196873710025102, 3.82194593839813e-05, 3.751534677576274e-05, 0.004743305500596762, 0.8406771421432495, 0.00018875481327995658, 0.00015232685836963356, 1.250763853022363e-05, 0.0002236300497315824, 0.0001328253565588966, 6.470848165918142e-05, 0.00013950491847936064, 9.763600246515125e-05, 0.0004387965309433639, 4.486108446144499e-05, 6.358909013215452e-05, 0.00034256128128618, 5.2226907428121194e-05, 5.293519507176825e-07, 4.435913979250472e-06, 6.543510244227946e-05, 2.9145055577828316e-06, 3.8726907405362e-06, 0.013709785416722298, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0077987853437662125, 0.00022834629635326564, 4.638521033939469e-07, 6.367737910295546e-07, 1.5816805898793973e-05, 4.74391745228786e-05, 5.2391105782589875e-06, 7.306586212507682e-06, 1.252215042768512e-05, 3.330249569444277e-07, 0.9909055233001709, 0.00045220632455311716, 5.365398919821018e-06, 7.535894837928936e-05, 8.799969691608567e-06, 7.895076123531908e-06, 0.00023918210354167968, 2.256911329823197e-06, 7.641898992005736e-05, 5.514193617273122e-05, 4.378300673124613e-06, 2.181093987019267e-05, 4.435156242266203e-08, 1.2584397381942836e-06, 2.069822357952944e-06, 1.2951757526025176e-05, 3.952872305035271e-07, 1.1202362202311633e-06, 7.981805538292974e-06, 2.9978705242683645e-06, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04228433594107628, 0.01130534615367651, 7.640720696144854e-07, 2.2586132217838895e-06, 9.786387818166986e-05, 0.03826921060681343, 0.00023640895960852504, 0.001455514575354755, 0.0029497891664505005, 1.2863508800364798e-06, 2.8271693736314774e-05, 0.8946183323860168, 0.0005235012504272163, 0.0011799612548202276, 0.0009106284123845398, 0.0003604615631047636, 0.00020461656094994396, 2.910211514972616e-05, 0.0013425340875983238, 0.0003398252301849425, 0.00033254953450523317, 8.430417801719159e-05, 7.937666879342942e-08, 5.80987489229301e-06, 4.91085984322126e-06, 0.002387275919318199, 2.696716364880558e-05, 0.00020931514154653996, 0.0007688859477639198, 5.801381576020503e-06, 3.4147800761274993e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0061029596254229546, 0.016550516709685326, 4.6485445636790246e-05, 8.233629245069096e-08, 7.398499292321503e-06, 0.0005482283304445446, 0.0005931411869823933, 1.5738387446617708e-05, 3.503288098727353e-05, 3.3504278462714865e-07, 1.8657722478110372e-07, 0.00046066963113844395, 0.8416223526000977, 0.12311305105686188, 0.006359235383570194, 0.0026995246298611164, 0.00016676251834724098, 0.001213077805005014, 0.00013198796659708023, 8.468224405078217e-05, 4.615935722540598e-06, 6.994968862272799e-05, 4.916098987450823e-06, 3.4761444567266153e-07, 3.7119539797458856e-07, 6.107360968599096e-05, 6.728529115207493e-05, 2.2758390514354687e-06, 2.0601644791895524e-05, 7.198171942945919e-07, 2.78813168108627e-08, 1.6529560525668785e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04041679948568344, 0.001346431439742446, 0.0002881708787754178, 4.343534783401992e-06, 0.0006837420514784753, 0.0021492524538189173, 0.0012713877949863672, 0.00021145616483408958, 0.0001491332659497857, 2.3681526727159508e-05, 9.255093027604744e-05, 7.460857159458101e-05, 0.0030381479300558567, 0.9153464436531067, 0.002063496969640255, 0.002670764224603772, 0.0006971358670853078, 0.022915009409189224, 0.0016386171337217093, 0.0002943731960840523, 8.301850357383955e-06, 0.0001194091746583581, 3.610825660871342e-05, 1.2593762221513316e-05, 0.00038307331851683557, 0.0011952053755521774, 0.0001844509970396757, 7.125815318431705e-05, 0.00012850550410803407, 6.062198372092098e-05, 4.517644993029535e-05, 6.895749606883328e-07, 0.002379077021032572, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.03775661438703537, 0.0016132340533658862, 0.0003177856560796499, 8.17116472262569e-07, 4.6315386725837016e-07, 7.222060230560601e-05, 0.00012490959488786757, 4.99756515637273e-06, 0.00030667142709717155, 5.749376668973127e-06, 0.00024661491625010967, 0.003777766600251198, 0.0013358162250369787, 0.002113183494657278, 0.8997159600257874, 0.04417850822210312, 0.00033094940590672195, 0.0013849412789568305, 0.00039355995249934494, 0.002321068197488785, 0.000464222306618467, 0.00019271834753453732, 0.0005703889764845371, 5.927758593315957e-06, 2.4687206945372964e-08, 3.1083311569091165e-06, 7.165991064539412e-06, 2.9240266030683415e-07, 7.984187686815858e-05, 1.2546481229946949e-05, 0.00012651427823584527, 0.00023838109336793423, 0.0008907323936000466, 0.0014062426052987576, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08771780878305435, 0.00782146118581295, 5.6624849094077945e-05, 1.4731161179781793e-08, 1.0490798558748793e-05, 0.0011058745440095663, 0.00042911627679131925, 5.52300589333754e-05, 3.919872688129544e-05, 2.7502203465701314e-06, 2.5461979475949192e-06, 0.002755212364718318, 0.005722680129110813, 0.0011191823286935687, 0.0015876379329711199, 0.8757358193397522, 0.001080039655789733, 0.0035321444738656282, 0.0015019761631265283, 0.0006949526141397655, 4.334090044721961e-05, 0.00023435342882294208, 6.934279099368723e-06, 2.757802519681718e-08, 9.912479299600818e-07, 0.00019156596681568772, 2.9515393180190586e-05, 5.468214112624992e-06, 2.480577677488327e-06, 1.4659925682281028e-06, 7.256068101924029e-07, 8.567866461817175e-05, 0.004263878334313631, 0.0007048743427731097, 0.0034579597413539886, 0.0, 0.0, 0.0, 0.0, 0.0], [0.018741386011242867, 0.0001317748101428151, 0.00011519038525875658, 8.336405130648927e-07, 3.873432774526009e-07, 4.0343722503166646e-05, 8.927313319873065e-05, 6.575927727681119e-06, 6.646558176726103e-05, 2.0145262169535272e-07, 0.00012498456635512412, 0.00022737613471690565, 0.00035967829171568155, 4.087335037183948e-05, 8.65399488247931e-05, 0.00017062197730410844, 0.9720673561096191, 0.004511047620326281, 0.0017741499468684196, 0.00033421028638258576, 0.0006673677125945687, 8.272402737929951e-06, 8.590817014919594e-05, 4.445200545433181e-07, 1.5395107411109166e-08, 3.916216883226298e-06, 1.8561571778263897e-05, 1.7440333976992406e-06, 2.306471833435353e-05, 6.561277814398636e-07, 5.536517346627079e-05, 1.0027780263044406e-05, 0.00011005577107425779, 2.0988292817492038e-05, 6.0086065786890686e-05, 4.435596929397434e-05, 0.0, 0.0, 0.0, 0.0], [0.02161449007689953, 6.699986261082813e-05, 0.00023383108782581985, 1.9461008093912824e-07, 1.05680453543755e-06, 2.569968273746781e-05, 0.00021046775509603322, 1.7352506347378949e-06, 6.270146968745394e-06, 8.374834692403965e-08, 2.354792513870052e-06, 1.806468026188668e-05, 0.00023376732133328915, 0.0018953320104628801, 0.00020409566059242934, 0.0004746932827401906, 0.0029953878838568926, 0.962553858757019, 0.0069142491556704044, 0.0001976252970052883, 0.00011761276255128905, 1.7810549252317287e-05, 8.193789835786447e-05, 8.319970135062249e-08, 6.341087299688297e-08, 1.5034723901408142e-06, 1.5951758541632444e-05, 1.285200426082156e-07, 7.935356052257703e-07, 1.2686608386047737e-07, 5.14019859565451e-07, 3.882940191601847e-08, 5.885375867364928e-05, 0.0011026912834495306, 9.71672561718151e-05, 8.231383981183171e-05, 0.0007721207803115249, 0.0, 0.0, 0.0], [0.036683376878499985, 5.292138666845858e-06, 1.6796333284219145e-06, 3.225949229967995e-10, 1.7679223418554102e-08, 9.711849997984245e-06, 6.97010455041891e-06, 3.922056293959031e-06, 1.5272446034941822e-05, 6.423179002013057e-07, 3.00263849339899e-07, 2.1527906937990338e-05, 8.620364724265528e-07, 2.37110707530519e-05, 1.133400132857787e-06, 4.3700861169782e-06, 2.1794843632960692e-05, 0.0007244828739203513, 0.956550121307373, 7.606161670992151e-05, 0.005361158866435289, 3.409740747883916e-05, 1.4788669204790494e-06, 1.967607188291254e-09, 6.515270101203896e-09, 7.016132030912559e-07, 1.9082037283624231e-07, 3.6050772678208887e-07, 3.973362709075445e-06, 1.5304426597140264e-06, 1.0371004321996224e-07, 7.635152314833249e-07, 1.8985033989338262e-07, 6.164461410662625e-06, 2.613814103824552e-06, 1.352905542262306e-06, 1.0274186024616938e-05, 0.0004238302353769541, 0.0, 0.0], [0.03937007114291191, 0.00035857962211593986, 7.419281610054895e-05, 4.950996662955731e-05, 4.6530603867722675e-06, 4.697596523328684e-05, 0.00022992271988186985, 1.2504364121923572e-06, 3.72327740478795e-05, 8.911823101698246e-08, 7.931985601317137e-05, 0.000135091133415699, 1.8374767023487948e-05, 0.00010297342669218779, 8.208450162783265e-05, 2.753637636487838e-05, 0.0005808338755741715, 0.0011750266421586275, 0.0009059829753823578, 0.9545682072639465, 7.417569577228278e-05, 0.0005086683668196201, 0.00019448318926151842, 8.633160177851096e-05, 8.95263099209842e-07, 1.4289948921941686e-05, 3.445189940975979e-05, 2.1258644267163618e-07, 1.6950663848547265e-05, 4.816170076082926e-07, 3.18538659485057e-05, 1.3589556147053372e-05, 2.5087487301789224e-05, 5.9819085436174646e-05, 0.0001169012684840709, 1.1341999197611585e-05, 0.0002713195572141558, 0.0002638357982505113, 0.00042732004658319056, 0.0], [0.025401176884770393, 7.441661864504567e-07, 1.0830090104718693e-05, 2.2497226304096785e-09, 9.221913788159952e-10, 8.208573490264826e-06, 2.5448041469644522e-06, 1.0004280426301193e-07, 3.160869891871698e-05, 2.4278301680169534e-07, 5.289186333357065e-07, 1.4262926924857311e-05, 1.5345807469202555e-06, 1.87778505278402e-06, 0.0002028680028161034, 2.1412402020359877e-06, 0.00010114445467479527, 0.0009309852030128241, 0.1016610637307167, 5.551521462621167e-05, 0.8627558350563049, 2.250279521831544e-06, 3.066586941713467e-05, 5.196949981467469e-09, 3.1991624627192294e-11, 8.736405021636529e-08, 1.2241659419487405e-07, 1.2374152813521277e-09, 1.2507486644608434e-05, 4.635763275473437e-07, 1.6297862259762042e-07, 2.719466181133612e-07, 1.1941818911509472e-07, 1.4462575848028791e-07, 5.527890243683942e-05, 3.5639450857161137e-07, 4.933351738145575e-05, 0.0001694788079475984, 0.008456701412796974, 3.871364606311545e-05]]]}\n",
        "    )\n",
        "    </script>"
       ],
       "text/plain": [
-       "<circuitsvis.utils.render.RenderedHTML at 0x7f2082f6d090>"
+       "<circuitsvis.utils.render.RenderedHTML at 0x7f2821c75090>"
       ]
      },
      "metadata": {},
@@ -1117,17 +1117,17 @@
    "execution_count": 21,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:42:15.119333Z",
-     "iopub.status.busy": "2024-09-05T17:42:15.118902Z",
-     "iopub.status.idle": "2024-09-05T17:42:20.805789Z",
-     "shell.execute_reply": "2024-09-05T17:42:20.805169Z"
+     "iopub.execute_input": "2024-09-10T17:15:53.547886Z",
+     "iopub.status.busy": "2024-09-10T17:15:53.547704Z",
+     "iopub.status.idle": "2024-09-10T17:15:55.834827Z",
+     "shell.execute_reply": "2024-09-10T17:15:55.834212Z"
     }
    },
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "43da9025ada5462c9ceacafb0040d9e5",
+       "model_id": "70c3c51f506a4de4974cf045dc40e642",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1141,7 +1141,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "b9efbbee427945918852a8f8351f3658",
+       "model_id": "c2024b930e0f46828f9f1935807c38fd",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1155,7 +1155,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "cad219cb55d34a84a8bd505e58157885",
+       "model_id": "0688c83589ce4772b743149fd2bfabce",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1169,7 +1169,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "e60d70eb78324541b8f9e1d0766491c8",
+       "model_id": "27f33facb9e34151b8dce0f220cad270",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1183,7 +1183,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "40e09b2a289644d4b6031bd159605d65",
+       "model_id": "f9a0ca4b543f47e0987f20fc87d7135a",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1197,7 +1197,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "8bccf23ea8614f7790636e7dd69b28b0",
+       "model_id": "e4a807366560448bab5fc25c6deea451",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1226,10 +1226,10 @@
    "execution_count": 22,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:42:20.808277Z",
-     "iopub.status.busy": "2024-09-05T17:42:20.807902Z",
-     "iopub.status.idle": "2024-09-05T17:42:21.432385Z",
-     "shell.execute_reply": "2024-09-05T17:42:21.431842Z"
+     "iopub.execute_input": "2024-09-10T17:15:55.837239Z",
+     "iopub.status.busy": "2024-09-10T17:15:55.836882Z",
+     "iopub.status.idle": "2024-09-10T17:15:56.450296Z",
+     "shell.execute_reply": "2024-09-10T17:15:56.449758Z"
     }
    },
    "outputs": [
@@ -1240,9 +1240,9 @@
        "<head><meta charset=\"utf-8\" /></head>\n",
        "<body>\n",
        "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
-       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"80cfda6e-58b4-4243-84ab-714b7a85733a\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"80cfda6e-58b4-4243-84ab-714b7a85733a\")) {                    Plotly.newPlot(                        \"80cfda6e-58b4-4243-84ab-714b7a85733a\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.010475716553628445,0.00012220541248098016,0.010917986743152142,7.656216439499985e-06,0.0006968218949623406,6.9313814492488746e-06,0.0084159504622221,0.0012861847644671798,0.008170148357748985,0.009381728246808052,0.008698969148099422,0.01542614120990038],[0.002591925673186779,0.01701531372964382,0.00246937433257699,0.00014253608242142946,0.014017364010214806,0.0016980216605588794,0.003427345072850585,0.015447111800312996,0.002018758561462164,0.0001262403529835865,0.003834882751107216,0.014691304415464401],[0.009293507784605026,0.006629703566431999,0.013332263566553593,0.0016038173343986273,0.019033372402191162,0.005262553226202726,0.009104946628212929,0.0015506644267588854,0.01918281428515911,0.012545435689389706,0.02117963135242462,4.0281530888641726e-10],[0.005776463076472282,0.22343513369560242,0.858056902885437,0.01594151370227337,0.016118988394737244,0.012186223641037941,0.019889742136001587,0.2014574259519577,0.01657862216234207,0.017072241753339767,0.935918390750885,0.4967102110385895],[0.2777242362499237,0.2320401519536972,0.09087100625038147,0.010066336952149868,0.08514240384101868,0.024723602458834648,0.645492672920227,0.024798421189188957,0.06863012909889221,0.655184268951416,0.015323049388825893,0.059535764157772064],[0.01975579373538494,0.07690141350030899,0.05422816053032875,0.01160797756165266,0.03362690284848213,0.1715722680091858,0.0725213959813118,0.09838436543941498,0.006914828438311815,0.4509258270263672,0.17343828082084656,0.030602477490901947]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Induction Score by Head in Distil GPT-2\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"7a2572ee-0c07-4383-838b-a1506ba287c1\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"7a2572ee-0c07-4383-838b-a1506ba287c1\")) {                    Plotly.newPlot(                        \"7a2572ee-0c07-4383-838b-a1506ba287c1\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.010156218893826008,0.0001236683747265488,0.011424270458519459,7.034143436612794e-06,0.000634899246506393,1.026747213472845e-05,0.008508010767400265,0.0013871823903173208,0.008700739592313766,0.009379616938531399,0.009299803525209427,0.015903957188129425],[0.0028657428920269012,0.015641430392861366,0.003444988513365388,0.0002817462373059243,0.013710059225559235,0.0021904385648667812,0.0045652128756046295,0.01538788340985775,0.00303249410353601,7.614650530740619e-05,0.003943697549402714,0.014629906043410301],[0.011179465800523758,0.007786225993186235,0.0122522609308362,0.001650955411605537,0.018960842862725258,0.004585776012390852,0.010203308425843716,0.0018027113983407617,0.017543736845254898,0.013146407902240753,0.021970190107822418,5.426245824197151e-12],[0.0065704951994121075,0.21819067001342773,0.8631006479263306,0.015738708898425102,0.017175206914544106,0.013265381567180157,0.01839497685432434,0.18099769949913025,0.014968963339924812,0.016049183905124664,0.9444844722747803,0.4698825478553772],[0.2739366590976715,0.2408347725868225,0.08438918739557266,0.011110114865005016,0.08356543630361557,0.024493325501680374,0.6308431625366211,0.024440230801701546,0.0658520832657814,0.6499699950218201,0.017077801749110222,0.05483752116560936],[0.019521579146385193,0.07837866246700287,0.05102621018886566,0.01296086423099041,0.032427575439214706,0.18894724547863007,0.07455243170261383,0.0973595529794693,0.007094149943441153,0.4619784951210022,0.1653374880552292,0.02855134755373001]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Induction Score by Head in Distil GPT-2\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('80cfda6e-58b4-4243-84ab-714b7a85733a');\n",
+       "var gd = document.getElementById('7a2572ee-0c07-4383-838b-a1506ba287c1');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -1424,10 +1424,10 @@
    "execution_count": 23,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:42:21.435297Z",
-     "iopub.status.busy": "2024-09-05T17:42:21.434950Z",
-     "iopub.status.idle": "2024-09-05T17:42:21.438890Z",
-     "shell.execute_reply": "2024-09-05T17:42:21.438175Z"
+     "iopub.execute_input": "2024-09-10T17:15:56.452600Z",
+     "iopub.status.busy": "2024-09-10T17:15:56.452247Z",
+     "iopub.status.idle": "2024-09-10T17:15:56.456303Z",
+     "shell.execute_reply": "2024-09-10T17:15:56.455832Z"
     }
    },
    "outputs": [
@@ -1468,10 +1468,10 @@
    "execution_count": 24,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:42:21.441073Z",
-     "iopub.status.busy": "2024-09-05T17:42:21.440720Z",
-     "iopub.status.idle": "2024-09-05T17:42:21.444258Z",
-     "shell.execute_reply": "2024-09-05T17:42:21.443711Z"
+     "iopub.execute_input": "2024-09-10T17:15:56.458300Z",
+     "iopub.status.busy": "2024-09-10T17:15:56.457993Z",
+     "iopub.status.idle": "2024-09-10T17:15:56.461757Z",
+     "shell.execute_reply": "2024-09-10T17:15:56.461307Z"
     }
    },
    "outputs": [
@@ -1513,10 +1513,10 @@
    "execution_count": 25,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:42:21.446336Z",
-     "iopub.status.busy": "2024-09-05T17:42:21.446005Z",
-     "iopub.status.idle": "2024-09-05T17:42:21.516719Z",
-     "shell.execute_reply": "2024-09-05T17:42:21.516200Z"
+     "iopub.execute_input": "2024-09-10T17:15:56.463804Z",
+     "iopub.status.busy": "2024-09-10T17:15:56.463523Z",
+     "iopub.status.idle": "2024-09-10T17:15:56.539633Z",
+     "shell.execute_reply": "2024-09-10T17:15:56.539039Z"
     }
    },
    "outputs": [
@@ -1610,10 +1610,10 @@
    "execution_count": 26,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:42:21.519165Z",
-     "iopub.status.busy": "2024-09-05T17:42:21.518776Z",
-     "iopub.status.idle": "2024-09-05T17:42:21.526605Z",
-     "shell.execute_reply": "2024-09-05T17:42:21.526104Z"
+     "iopub.execute_input": "2024-09-10T17:15:56.542117Z",
+     "iopub.status.busy": "2024-09-10T17:15:56.541778Z",
+     "iopub.status.idle": "2024-09-10T17:15:56.549268Z",
+     "shell.execute_reply": "2024-09-10T17:15:56.548714Z"
     }
    },
    "outputs": [],
@@ -1627,10 +1627,10 @@
    "execution_count": 27,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:42:21.528767Z",
-     "iopub.status.busy": "2024-09-05T17:42:21.528421Z",
-     "iopub.status.idle": "2024-09-05T17:42:21.533866Z",
-     "shell.execute_reply": "2024-09-05T17:42:21.533308Z"
+     "iopub.execute_input": "2024-09-10T17:15:56.551463Z",
+     "iopub.status.busy": "2024-09-10T17:15:56.551058Z",
+     "iopub.status.idle": "2024-09-10T17:15:56.556549Z",
+     "shell.execute_reply": "2024-09-10T17:15:56.556074Z"
     }
    },
    "outputs": [
@@ -1708,10 +1708,10 @@
    "execution_count": 28,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:42:21.535914Z",
-     "iopub.status.busy": "2024-09-05T17:42:21.535597Z",
-     "iopub.status.idle": "2024-09-05T17:42:21.540031Z",
-     "shell.execute_reply": "2024-09-05T17:42:21.539573Z"
+     "iopub.execute_input": "2024-09-10T17:15:56.558629Z",
+     "iopub.status.busy": "2024-09-10T17:15:56.558309Z",
+     "iopub.status.idle": "2024-09-10T17:15:56.562453Z",
+     "shell.execute_reply": "2024-09-10T17:15:56.562023Z"
     }
    },
    "outputs": [
@@ -1776,10 +1776,10 @@
    "execution_count": 29,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:42:21.542069Z",
-     "iopub.status.busy": "2024-09-05T17:42:21.541763Z",
-     "iopub.status.idle": "2024-09-05T17:42:21.545937Z",
-     "shell.execute_reply": "2024-09-05T17:42:21.545381Z"
+     "iopub.execute_input": "2024-09-10T17:15:56.564630Z",
+     "iopub.status.busy": "2024-09-10T17:15:56.564316Z",
+     "iopub.status.idle": "2024-09-10T17:15:56.568181Z",
+     "shell.execute_reply": "2024-09-10T17:15:56.567713Z"
     }
    },
    "outputs": [
@@ -1809,10 +1809,10 @@
    "execution_count": 30,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:42:21.548006Z",
-     "iopub.status.busy": "2024-09-05T17:42:21.547616Z",
-     "iopub.status.idle": "2024-09-05T17:42:21.551026Z",
-     "shell.execute_reply": "2024-09-05T17:42:21.550597Z"
+     "iopub.execute_input": "2024-09-10T17:15:56.570168Z",
+     "iopub.status.busy": "2024-09-10T17:15:56.569846Z",
+     "iopub.status.idle": "2024-09-10T17:15:56.573297Z",
+     "shell.execute_reply": "2024-09-10T17:15:56.572869Z"
     }
    },
    "outputs": [
@@ -1849,10 +1849,10 @@
    "execution_count": 31,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:42:21.553103Z",
-     "iopub.status.busy": "2024-09-05T17:42:21.552675Z",
-     "iopub.status.idle": "2024-09-05T17:42:21.556466Z",
-     "shell.execute_reply": "2024-09-05T17:42:21.555906Z"
+     "iopub.execute_input": "2024-09-10T17:15:56.575326Z",
+     "iopub.status.busy": "2024-09-10T17:15:56.575006Z",
+     "iopub.status.idle": "2024-09-10T17:15:56.578860Z",
+     "shell.execute_reply": "2024-09-10T17:15:56.578300Z"
     }
    },
    "outputs": [
@@ -1892,10 +1892,10 @@
    "execution_count": 32,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:42:21.558320Z",
-     "iopub.status.busy": "2024-09-05T17:42:21.558157Z",
-     "iopub.status.idle": "2024-09-05T17:42:21.652451Z",
-     "shell.execute_reply": "2024-09-05T17:42:21.651922Z"
+     "iopub.execute_input": "2024-09-10T17:15:56.580911Z",
+     "iopub.status.busy": "2024-09-10T17:15:56.580601Z",
+     "iopub.status.idle": "2024-09-10T17:15:56.669916Z",
+     "shell.execute_reply": "2024-09-10T17:15:56.669306Z"
     }
    },
    "outputs": [
@@ -1903,14 +1903,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Probability tensor shape [batch, position, d_vocab] == torch.Size([1, 8, 50257])"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "\n",
+      "Probability tensor shape [batch, position, d_vocab] == torch.Size([1, 8, 50257])\n",
       "| The| probability: 11.98%\n"
      ]
     }
@@ -1939,10 +1932,10 @@
    "execution_count": 33,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:42:21.654915Z",
-     "iopub.status.busy": "2024-09-05T17:42:21.654567Z",
-     "iopub.status.idle": "2024-09-05T17:42:21.658318Z",
-     "shell.execute_reply": "2024-09-05T17:42:21.657841Z"
+     "iopub.execute_input": "2024-09-10T17:15:56.672288Z",
+     "iopub.status.busy": "2024-09-10T17:15:56.671905Z",
+     "iopub.status.idle": "2024-09-10T17:15:56.675936Z",
+     "shell.execute_reply": "2024-09-10T17:15:56.675464Z"
     }
    },
    "outputs": [
@@ -1977,10 +1970,10 @@
    "execution_count": 34,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:42:21.660139Z",
-     "iopub.status.busy": "2024-09-05T17:42:21.659969Z",
-     "iopub.status.idle": "2024-09-05T17:42:21.664282Z",
-     "shell.execute_reply": "2024-09-05T17:42:21.663826Z"
+     "iopub.execute_input": "2024-09-10T17:15:56.678017Z",
+     "iopub.status.busy": "2024-09-10T17:15:56.677689Z",
+     "iopub.status.idle": "2024-09-10T17:15:56.682040Z",
+     "shell.execute_reply": "2024-09-10T17:15:56.681495Z"
     }
    },
    "outputs": [
@@ -2010,10 +2003,10 @@
    "execution_count": 35,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:42:21.666206Z",
-     "iopub.status.busy": "2024-09-05T17:42:21.666038Z",
-     "iopub.status.idle": "2024-09-05T17:42:21.670364Z",
-     "shell.execute_reply": "2024-09-05T17:42:21.669896Z"
+     "iopub.execute_input": "2024-09-10T17:15:56.684219Z",
+     "iopub.status.busy": "2024-09-10T17:15:56.683829Z",
+     "iopub.status.idle": "2024-09-10T17:15:56.688030Z",
+     "shell.execute_reply": "2024-09-10T17:15:56.687475Z"
     }
    },
    "outputs": [
@@ -2049,10 +2042,10 @@
    "execution_count": 36,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:42:21.672326Z",
-     "iopub.status.busy": "2024-09-05T17:42:21.672000Z",
-     "iopub.status.idle": "2024-09-05T17:42:21.675992Z",
-     "shell.execute_reply": "2024-09-05T17:42:21.675552Z"
+     "iopub.execute_input": "2024-09-10T17:15:56.690101Z",
+     "iopub.status.busy": "2024-09-10T17:15:56.689800Z",
+     "iopub.status.idle": "2024-09-10T17:15:56.693763Z",
+     "shell.execute_reply": "2024-09-10T17:15:56.693150Z"
     }
    },
    "outputs": [
@@ -2100,10 +2093,10 @@
    "execution_count": 37,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:42:21.678024Z",
-     "iopub.status.busy": "2024-09-05T17:42:21.677712Z",
-     "iopub.status.idle": "2024-09-05T17:42:21.892205Z",
-     "shell.execute_reply": "2024-09-05T17:42:21.891572Z"
+     "iopub.execute_input": "2024-09-10T17:15:56.695971Z",
+     "iopub.status.busy": "2024-09-10T17:15:56.695807Z",
+     "iopub.status.idle": "2024-09-10T17:15:56.912760Z",
+     "shell.execute_reply": "2024-09-10T17:15:56.912075Z"
     }
    },
    "outputs": [
@@ -2153,10 +2146,10 @@
    "execution_count": 38,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:42:21.894842Z",
-     "iopub.status.busy": "2024-09-05T17:42:21.894368Z",
-     "iopub.status.idle": "2024-09-05T17:42:22.097813Z",
-     "shell.execute_reply": "2024-09-05T17:42:22.097243Z"
+     "iopub.execute_input": "2024-09-10T17:15:56.915302Z",
+     "iopub.status.busy": "2024-09-10T17:15:56.914930Z",
+     "iopub.status.idle": "2024-09-10T17:15:57.128814Z",
+     "shell.execute_reply": "2024-09-10T17:15:57.128220Z"
     }
    },
    "outputs": [
@@ -2199,10 +2192,10 @@
    "execution_count": 39,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:42:22.100256Z",
-     "iopub.status.busy": "2024-09-05T17:42:22.099915Z",
-     "iopub.status.idle": "2024-09-05T17:42:22.103964Z",
-     "shell.execute_reply": "2024-09-05T17:42:22.103518Z"
+     "iopub.execute_input": "2024-09-10T17:15:57.131296Z",
+     "iopub.status.busy": "2024-09-10T17:15:57.130871Z",
+     "iopub.status.idle": "2024-09-10T17:15:57.134979Z",
+     "shell.execute_reply": "2024-09-10T17:15:57.134441Z"
     }
    },
    "outputs": [
@@ -2259,10 +2252,10 @@
    "execution_count": 40,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:42:22.106017Z",
-     "iopub.status.busy": "2024-09-05T17:42:22.105682Z",
-     "iopub.status.idle": "2024-09-05T17:42:22.111387Z",
-     "shell.execute_reply": "2024-09-05T17:42:22.110938Z"
+     "iopub.execute_input": "2024-09-10T17:15:57.137209Z",
+     "iopub.status.busy": "2024-09-10T17:15:57.136899Z",
+     "iopub.status.idle": "2024-09-10T17:15:57.142506Z",
+     "shell.execute_reply": "2024-09-10T17:15:57.142012Z"
     }
    },
    "outputs": [
@@ -2304,10 +2297,10 @@
    "execution_count": 41,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:42:22.113417Z",
-     "iopub.status.busy": "2024-09-05T17:42:22.113075Z",
-     "iopub.status.idle": "2024-09-05T17:42:22.118235Z",
-     "shell.execute_reply": "2024-09-05T17:42:22.117792Z"
+     "iopub.execute_input": "2024-09-10T17:15:57.144533Z",
+     "iopub.status.busy": "2024-09-10T17:15:57.144226Z",
+     "iopub.status.idle": "2024-09-10T17:15:57.149411Z",
+     "shell.execute_reply": "2024-09-10T17:15:57.148957Z"
     }
    },
    "outputs": [
@@ -2349,10 +2342,10 @@
    "execution_count": 42,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:42:22.120251Z",
-     "iopub.status.busy": "2024-09-05T17:42:22.119920Z",
-     "iopub.status.idle": "2024-09-05T17:42:22.125302Z",
-     "shell.execute_reply": "2024-09-05T17:42:22.124814Z"
+     "iopub.execute_input": "2024-09-10T17:15:57.151365Z",
+     "iopub.status.busy": "2024-09-10T17:15:57.151038Z",
+     "iopub.status.idle": "2024-09-10T17:15:57.156341Z",
+     "shell.execute_reply": "2024-09-10T17:15:57.155853Z"
     }
    },
    "outputs": [
@@ -2391,10 +2384,10 @@
    "execution_count": 43,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:42:22.127451Z",
-     "iopub.status.busy": "2024-09-05T17:42:22.127085Z",
-     "iopub.status.idle": "2024-09-05T17:42:22.130905Z",
-     "shell.execute_reply": "2024-09-05T17:42:22.130369Z"
+     "iopub.execute_input": "2024-09-10T17:15:57.158451Z",
+     "iopub.status.busy": "2024-09-10T17:15:57.158089Z",
+     "iopub.status.idle": "2024-09-10T17:15:57.161790Z",
+     "shell.execute_reply": "2024-09-10T17:15:57.161293Z"
     }
    },
    "outputs": [
@@ -2441,10 +2434,10 @@
    "execution_count": 44,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:42:22.133086Z",
-     "iopub.status.busy": "2024-09-05T17:42:22.132755Z",
-     "iopub.status.idle": "2024-09-05T17:42:22.140103Z",
-     "shell.execute_reply": "2024-09-05T17:42:22.139570Z"
+     "iopub.execute_input": "2024-09-10T17:15:57.163768Z",
+     "iopub.status.busy": "2024-09-10T17:15:57.163561Z",
+     "iopub.status.idle": "2024-09-10T17:15:57.170358Z",
+     "shell.execute_reply": "2024-09-10T17:15:57.169848Z"
     }
    },
    "outputs": [
@@ -2466,10 +2459,10 @@
    "execution_count": 45,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:42:22.142375Z",
-     "iopub.status.busy": "2024-09-05T17:42:22.142002Z",
-     "iopub.status.idle": "2024-09-05T17:42:22.259008Z",
-     "shell.execute_reply": "2024-09-05T17:42:22.258318Z"
+     "iopub.execute_input": "2024-09-10T17:15:57.172664Z",
+     "iopub.status.busy": "2024-09-10T17:15:57.172292Z",
+     "iopub.status.idle": "2024-09-10T17:15:57.290945Z",
+     "shell.execute_reply": "2024-09-10T17:15:57.290346Z"
     }
    },
    "outputs": [
@@ -2493,10 +2486,10 @@
    "execution_count": 46,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:42:22.261417Z",
-     "iopub.status.busy": "2024-09-05T17:42:22.261007Z",
-     "iopub.status.idle": "2024-09-05T17:42:22.290961Z",
-     "shell.execute_reply": "2024-09-05T17:42:22.290449Z"
+     "iopub.execute_input": "2024-09-10T17:15:57.293299Z",
+     "iopub.status.busy": "2024-09-10T17:15:57.293127Z",
+     "iopub.status.idle": "2024-09-10T17:15:57.322911Z",
+     "shell.execute_reply": "2024-09-10T17:15:57.322454Z"
     }
    },
    "outputs": [
@@ -2507,9 +2500,9 @@
        "<head><meta charset=\"utf-8\" /></head>\n",
        "<body>\n",
        "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
-       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"f0e72e2d-ef0f-47a5-a93a-ab60fc607f1b\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"f0e72e2d-ef0f-47a5-a93a-ab60fc607f1b\")) {                    Plotly.newPlot(                        \"f0e72e2d-ef0f-47a5-a93a-ab60fc607f1b\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.7775008678436279,0.35272663831710815,0.25961872935295105,0.6670258045196533,0.838425874710083,0.5584433078765869,0.844474732875824,0.41379114985466003,0.244889497756958,0.02815753035247326,0.3584096431732178,0.16288283467292786],[-0.4541913866996765,-0.6529324650764465,-0.5484569668769836,-0.7990368604660034,-0.7736426591873169,-0.8522582650184631,0.977432370185852,0.6626251935958862,-0.7303223609924316,-0.7007019519805908,-0.6946623921394348,-0.9996723532676697],[-0.7837162613868713,0.8967758417129517,0.4750956892967224,-0.6671973466873169,0.7881463170051575,-0.8547748923301697,-0.9054183959960938,-0.5749384760856628,-0.321751207113266,-0.0285941194742918,-0.9247617125511169,-0.9699268937110901],[0.5864036083221436,-0.76143479347229,0.5971695184707642,0.7854391932487488,-0.8788884878158569,0.3908745348453522,0.0447387658059597,0.1102800965309143,-0.8169987201690674,0.2212953418493271,-0.9939578771591187,0.5774401426315308],[0.5254791378974915,0.30490121245384216,-0.10729170590639114,0.9433152675628662,-0.9314427971839905,0.5273631811141968,-0.4264712929725647,-0.9984429478645325,0.5296757817268372,0.8604294657707214,-0.8895052075386047,0.9556970000267029],[0.6629188656806946,0.4295697808265686,0.9736858010292053,0.655548095703125,0.12201863527297974,0.7442769408226013,0.5037954449653625,0.95253586769104,-0.6507166624069214,-0.9316278100013733,0.9791510701179504,-0.9972586035728455],[0.9613031148910522,0.7501779794692993,-0.3806658983230591,0.6429785490036011,0.9557769298553467,-0.9428837895393372,-0.9948079586029053,0.7852989435195923,0.9657301306724548,0.7073014974594116,0.36872273683547974,0.8128010034561157],[0.9659482836723328,0.9730120301246643,0.31900620460510254,-0.30290529131889343,0.9790952801704407,0.9357922673225403,-0.5550314784049988,-0.0054661668837070465,0.986777663230896,0.8249568343162537,0.5664296746253967,0.1000528484582901],[-0.9464486837387085,-0.25471991300582886,0.6522327065467834,0.14152583479881287,0.9884141683578491,0.9860584735870361,0.6949271559715271,0.9901811480522156,0.9791203141212463,-0.23595543205738068,-0.982071042060852,0.6506688594818115],[0.9895945191383362,-0.291781485080719,0.9714024662971497,0.9951602220535278,0.18783727288246155,-0.9460937976837158,0.4780191481113434,-0.2489192634820938,0.9437099099159241,0.11866225302219391,0.9941242933273315,-0.38088199496269226],[0.9564487338066101,0.5542722344398499,0.42118069529533386,0.6628788113594055,0.8659593462944031,0.9937117695808411,0.9069075584411621,0.3981107473373413,-0.4134218096733093,0.9971914887428284,0.3459664583206177,0.9938657283782959],[0.5891268253326416,0.9313738942146301,0.9268401861190796,0.9993563890457153,0.6227542161941528,0.8463947772979736,0.6584343910217285,0.8423123955726624,0.2978499233722687,0.8728678822517395,0.9963143467903137,0.9867526292800903]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0,\"cmin\":-1.0,\"cmax\":1.0},\"title\":{\"text\":\"OV Copying Score for each head in GPT-2 Small\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"83f1fdc6-5e71-4ccb-8cbf-7af2e7ebf238\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"83f1fdc6-5e71-4ccb-8cbf-7af2e7ebf238\")) {                    Plotly.newPlot(                        \"83f1fdc6-5e71-4ccb-8cbf-7af2e7ebf238\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.7775008678436279,0.35272663831710815,0.25961872935295105,0.6670258045196533,0.838425874710083,0.5584433078765869,0.844474732875824,0.41379114985466003,0.244889497756958,0.02815753035247326,0.3584096431732178,0.16288283467292786],[-0.4541913866996765,-0.6529324650764465,-0.5484569668769836,-0.7990368604660034,-0.7736426591873169,-0.8522582650184631,0.977432370185852,0.6626251935958862,-0.7303223609924316,-0.7007019519805908,-0.6946623921394348,-0.9996723532676697],[-0.7837162613868713,0.8967758417129517,0.4750956892967224,-0.6671973466873169,0.7881463170051575,-0.8547748923301697,-0.9054183959960938,-0.5749384760856628,-0.321751207113266,-0.0285941194742918,-0.9247617125511169,-0.9699268937110901],[0.5864036083221436,-0.76143479347229,0.5971695184707642,0.7854391932487488,-0.8788884878158569,0.3908745348453522,0.0447387658059597,0.1102800965309143,-0.8169987201690674,0.2212953418493271,-0.9939578771591187,0.5774401426315308],[0.5254791378974915,0.30490121245384216,-0.10729170590639114,0.9433152675628662,-0.9314427971839905,0.5273631811141968,-0.4264712929725647,-0.9984429478645325,0.5296757817268372,0.8604294657707214,-0.8895052075386047,0.9556970000267029],[0.6629188656806946,0.4295697808265686,0.9736858010292053,0.655548095703125,0.12201863527297974,0.7442769408226013,0.5037954449653625,0.95253586769104,-0.6507166624069214,-0.9316278100013733,0.9791510701179504,-0.9972586035728455],[0.9613031148910522,0.7501779794692993,-0.3806658983230591,0.6429785490036011,0.9557769298553467,-0.9428837895393372,-0.9948079586029053,0.7852989435195923,0.9657301306724548,0.7073014974594116,0.36872273683547974,0.8128010034561157],[0.9659482836723328,0.9730120301246643,0.31900620460510254,-0.30290529131889343,0.9790952801704407,0.9357922673225403,-0.5550314784049988,-0.0054661668837070465,0.986777663230896,0.8249568343162537,0.5664296746253967,0.1000528484582901],[-0.9464486837387085,-0.25471991300582886,0.6522327065467834,0.14152583479881287,0.9884141683578491,0.9860584735870361,0.6949271559715271,0.9901811480522156,0.9791203141212463,-0.23595543205738068,-0.982071042060852,0.6506688594818115],[0.9895945191383362,-0.291781485080719,0.9714024662971497,0.9951602220535278,0.18783727288246155,-0.9460937976837158,0.4780191481113434,-0.2489192634820938,0.9437099099159241,0.11866225302219391,0.9941242933273315,-0.38088199496269226],[0.9564487338066101,0.5542722344398499,0.42118069529533386,0.6628788113594055,0.8659593462944031,0.9937117695808411,0.9069075584411621,0.3981107473373413,-0.4134218096733093,0.9971914887428284,0.3459664583206177,0.9938657283782959],[0.5891268253326416,0.9313738942146301,0.9268401861190796,0.9993563890457153,0.6227542161941528,0.8463947772979736,0.6584343910217285,0.8423123955726624,0.2978499233722687,0.8728678822517395,0.9963143467903137,0.9867526292800903]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0,\"cmin\":-1.0,\"cmax\":1.0},\"title\":{\"text\":\"OV Copying Score for each head in GPT-2 Small\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('f0e72e2d-ef0f-47a5-a93a-ab60fc607f1b');\n",
+       "var gd = document.getElementById('83f1fdc6-5e71-4ccb-8cbf-7af2e7ebf238');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -2557,10 +2550,10 @@
    "execution_count": 47,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:42:22.293237Z",
-     "iopub.status.busy": "2024-09-05T17:42:22.292871Z",
-     "iopub.status.idle": "2024-09-05T17:42:22.323847Z",
-     "shell.execute_reply": "2024-09-05T17:42:22.323304Z"
+     "iopub.execute_input": "2024-09-10T17:15:57.324961Z",
+     "iopub.status.busy": "2024-09-10T17:15:57.324666Z",
+     "iopub.status.idle": "2024-09-10T17:15:57.353993Z",
+     "shell.execute_reply": "2024-09-10T17:15:57.353447Z"
     }
    },
    "outputs": [
@@ -2571,9 +2564,9 @@
        "<head><meta charset=\"utf-8\" /></head>\n",
        "<body>\n",
        "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
-       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"eb9ce2dc-9731-481d-a530-6a91763cefe7\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"eb9ce2dc-9731-481d-a530-6a91763cefe7\")) {                    Plotly.newPlot(                        \"eb9ce2dc-9731-481d-a530-6a91763cefe7\",                        [{\"hovertemplate\":\"Real=%{x}\\u003cbr\\u003eImaginary=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"\",\"marker\":{\"color\":\"#636efa\",\"symbol\":\"circle\"},\"mode\":\"markers\",\"name\":\"\",\"orientation\":\"v\",\"showlegend\":false,\"x\":[-2.139726161956787,1.4152659177780151,3.4444546699523926,4.0276689529418945,8.882657051086426,4.866776943206787,4.866776943206787,4.843708038330078,4.843708038330078,8.477534294128418,8.21679973602295,8.21679973602295,5.078621864318848,7.855459213256836,7.855459213256836,5.365771770477295,5.365771770477295,5.563427448272705,5.563427448272705,5.421727657318115,7.769131660461426,7.769131660461426,7.042290210723877,7.042290210723877,5.675149917602539,5.675149917602539,7.6785993576049805,7.6785993576049805,6.573331356048584,6.573331356048584,7.67292594909668,7.172202110290527,7.172202110290527,7.423615455627441,7.423615455627441,7.4708147048950195,6.08908748626709,6.08908748626709,6.306832313537598,6.306832313537598,6.511747360229492,6.511747360229492,5.955250263214111,5.955250263214111,5.858802795410156,5.858802795410156,7.147878170013428,7.147878170013428,7.185699939727783,7.185699939727783,6.6706037521362305,6.6706037521362305,6.7359819412231445,6.7359819412231445,6.149754524230957,6.149754524230957,6.288780212402344,6.288780212402344,6.344789028167725,6.625576972961426,6.625576972961426,6.899187088012695,6.899187088012695,6.856410026550293],\"xaxis\":\"x\",\"y\":[0.0,0.0,0.0,0.0,0.0,0.41851967573165894,-0.41851967573165894,0.09079498052597046,-0.09079498052597046,0.0,0.40868431329727173,-0.40868431329727173,0.0,0.7007191777229309,-0.7007191777229309,0.46421778202056885,-0.46421778202056885,0.5558270215988159,-0.5558270215988159,0.0,0.47056713700294495,-0.47056713700294495,1.029872179031372,-1.029872179031372,0.48252731561660767,-0.48252731561660767,0.33564886450767517,-0.33564886450767517,0.9988701343536377,-0.9988701343536377,0.0,0.7531797885894775,-0.7531797885894775,0.425758957862854,-0.425758957862854,0.0,0.6436269283294678,-0.6436269283294678,0.7701666951179504,-0.7701666951179504,0.7558029294013977,-0.7558029294013977,0.2591177523136139,-0.2591177523136139,0.013043581508100033,-0.013043581508100033,0.40166139602661133,-0.40166139602661133,0.28192487359046936,-0.28192487359046936,0.6146230697631836,-0.6146230697631836,0.5391324758529663,-0.5391324758529663,0.2823374569416046,-0.2823374569416046,0.35283783078193665,-0.35283783078193665,0.0,0.24868342280387878,-0.24868342280387878,0.15545885264873505,-0.15545885264873505,0.0],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Real\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Imaginary\"}},\"legend\":{\"tracegroupgap\":0},\"title\":{\"text\":\"Eigenvalues of Head L11H11 of GPT-2 Small\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"25ecddbf-c1bd-497d-b838-6c0e162ebacb\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"25ecddbf-c1bd-497d-b838-6c0e162ebacb\")) {                    Plotly.newPlot(                        \"25ecddbf-c1bd-497d-b838-6c0e162ebacb\",                        [{\"hovertemplate\":\"Real=%{x}\\u003cbr\\u003eImaginary=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"\",\"marker\":{\"color\":\"#636efa\",\"symbol\":\"circle\"},\"mode\":\"markers\",\"name\":\"\",\"orientation\":\"v\",\"showlegend\":false,\"x\":[-2.139726161956787,1.4152659177780151,3.4444546699523926,4.0276689529418945,8.882657051086426,4.866776943206787,4.866776943206787,4.843708038330078,4.843708038330078,8.477534294128418,8.21679973602295,8.21679973602295,5.078621864318848,7.855459213256836,7.855459213256836,5.365771770477295,5.365771770477295,5.563427448272705,5.563427448272705,5.421727657318115,7.769131660461426,7.769131660461426,7.042290210723877,7.042290210723877,5.675149917602539,5.675149917602539,7.6785993576049805,7.6785993576049805,6.573331356048584,6.573331356048584,7.67292594909668,7.172202110290527,7.172202110290527,7.423615455627441,7.423615455627441,7.4708147048950195,6.08908748626709,6.08908748626709,6.306832313537598,6.306832313537598,6.511747360229492,6.511747360229492,5.955250263214111,5.955250263214111,5.858802795410156,5.858802795410156,7.147878170013428,7.147878170013428,7.185699939727783,7.185699939727783,6.6706037521362305,6.6706037521362305,6.7359819412231445,6.7359819412231445,6.149754524230957,6.149754524230957,6.288780212402344,6.288780212402344,6.344789028167725,6.625576972961426,6.625576972961426,6.899187088012695,6.899187088012695,6.856410026550293],\"xaxis\":\"x\",\"y\":[0.0,0.0,0.0,0.0,0.0,0.41851967573165894,-0.41851967573165894,0.09079498052597046,-0.09079498052597046,0.0,0.40868431329727173,-0.40868431329727173,0.0,0.7007191777229309,-0.7007191777229309,0.46421778202056885,-0.46421778202056885,0.5558270215988159,-0.5558270215988159,0.0,0.47056713700294495,-0.47056713700294495,1.029872179031372,-1.029872179031372,0.48252731561660767,-0.48252731561660767,0.33564886450767517,-0.33564886450767517,0.9988701343536377,-0.9988701343536377,0.0,0.7531797885894775,-0.7531797885894775,0.425758957862854,-0.425758957862854,0.0,0.6436269283294678,-0.6436269283294678,0.7701666951179504,-0.7701666951179504,0.7558029294013977,-0.7558029294013977,0.2591177523136139,-0.2591177523136139,0.013043581508100033,-0.013043581508100033,0.40166139602661133,-0.40166139602661133,0.28192487359046936,-0.28192487359046936,0.6146230697631836,-0.6146230697631836,0.5391324758529663,-0.5391324758529663,0.2823374569416046,-0.2823374569416046,0.35283783078193665,-0.35283783078193665,0.0,0.24868342280387878,-0.24868342280387878,0.15545885264873505,-0.15545885264873505,0.0],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Real\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Imaginary\"}},\"legend\":{\"tracegroupgap\":0},\"title\":{\"text\":\"Eigenvalues of Head L11H11 of GPT-2 Small\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('eb9ce2dc-9731-481d-a530-6a91763cefe7');\n",
+       "var gd = document.getElementById('25ecddbf-c1bd-497d-b838-6c0e162ebacb');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -2620,10 +2613,10 @@
    "execution_count": 48,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:42:22.326197Z",
-     "iopub.status.busy": "2024-09-05T17:42:22.325766Z",
-     "iopub.status.idle": "2024-09-05T17:42:32.749968Z",
-     "shell.execute_reply": "2024-09-05T17:42:32.749351Z"
+     "iopub.execute_input": "2024-09-10T17:15:57.356134Z",
+     "iopub.status.busy": "2024-09-10T17:15:57.355970Z",
+     "iopub.status.idle": "2024-09-10T17:16:07.931260Z",
+     "shell.execute_reply": "2024-09-10T17:16:07.930733Z"
     }
    },
    "outputs": [
@@ -2645,10 +2638,10 @@
    "execution_count": 49,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:42:32.752606Z",
-     "iopub.status.busy": "2024-09-05T17:42:32.752168Z",
-     "iopub.status.idle": "2024-09-05T17:42:33.291006Z",
-     "shell.execute_reply": "2024-09-05T17:42:33.290373Z"
+     "iopub.execute_input": "2024-09-10T17:16:07.933786Z",
+     "iopub.status.busy": "2024-09-10T17:16:07.933431Z",
+     "iopub.status.idle": "2024-09-10T17:16:08.475037Z",
+     "shell.execute_reply": "2024-09-10T17:16:08.474424Z"
     }
    },
    "outputs": [
@@ -2672,10 +2665,10 @@
    "execution_count": 50,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:42:33.293365Z",
-     "iopub.status.busy": "2024-09-05T17:42:33.293028Z",
-     "iopub.status.idle": "2024-09-05T17:42:33.322298Z",
-     "shell.execute_reply": "2024-09-05T17:42:33.321717Z"
+     "iopub.execute_input": "2024-09-10T17:16:08.477334Z",
+     "iopub.status.busy": "2024-09-10T17:16:08.477117Z",
+     "iopub.status.idle": "2024-09-10T17:16:08.507444Z",
+     "shell.execute_reply": "2024-09-10T17:16:08.506910Z"
     }
    },
    "outputs": [
@@ -2686,9 +2679,9 @@
        "<head><meta charset=\"utf-8\" /></head>\n",
        "<body>\n",
        "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
-       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"e3552224-0615-4466-b566-225b4371075b\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"e3552224-0615-4466-b566-225b4371075b\")) {                    Plotly.newPlot(                        \"e3552224-0615-4466-b566-225b4371075b\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.8356367349624634,0.5853534936904907,0.5105841159820557,0.7843377590179443,0.8644161224365234,0.7026589512825012,0.8969924449920654,0.5868823528289795,0.42486509680747986,-0.16337518393993378,0.46268585324287415,0.276053786277771],[-0.052920110523700714,-0.31773144006729126,-0.4810579717159271,-0.7838066220283508,-0.6360211968421936,-0.77586829662323,0.9681803584098816,0.8119116425514221,-0.7510464787483215,-0.6878445744514465,-0.6429887413978577,-0.9985856413841248],[-0.6598325967788696,0.9152501821517944,0.5461500883102417,-0.48743969202041626,0.7720563411712646,-0.7541061639785767,-0.8472450971603394,-0.6948987245559692,-0.1557510495185852,0.2444225549697876,-0.9106622338294983,-0.9439151287078857],[0.648689866065979,-0.5592910647392273,0.5935593247413635,0.7843040823936462,-0.8150346279144287,0.613004744052887,0.16785882413387299,0.35195910930633545,-0.6837262511253357,0.2223764955997467,-0.9929219484329224,0.6535818576812744],[0.5740953087806702,0.3640134036540985,0.09609051048755646,0.9359624981880188,-0.9228776097297668,0.6191076636314392,-0.33572638034820557,-0.998464822769165,0.6448632478713989,0.8468661308288574,-0.7557656764984131,0.9527971744537354],[0.7326544523239136,0.5324169397354126,0.9732670187950134,0.7239246964454651,0.2553895115852356,0.8158416152000427,0.665579080581665,0.9287100434303284,-0.5660436153411865,-0.8908745050430298,0.9834234118461609,-0.9981179237365723],[0.9698692560195923,0.7439671754837036,-0.35639333724975586,0.6022987365722656,0.9708116054534912,-0.9278275966644287,-0.9962316155433655,0.834520697593689,0.9714328050613403,0.8158544898033142,0.5902575850486755,0.8199343681335449],[0.9820227026939392,0.9859329462051392,0.5152460932731628,-0.5610517263412476,0.9663666486740112,0.9495157599449158,-0.5204814076423645,0.3104752004146576,0.9859083890914917,0.7797460556030273,0.6738530397415161,0.39197394251823425],[-0.906204104423523,0.11750960350036621,0.8077874779701233,0.416930615901947,0.9829014539718628,0.9902300834655762,0.7847104668617249,0.9945629835128784,0.9868024587631226,-0.26804426312446594,-0.9908867478370667,0.745792806148529],[0.9906190633773804,-0.18231146037578583,0.9757838249206543,0.9986750483512878,0.2544330954551697,-0.954406201839447,0.586924135684967,-0.23537978529930115,0.9550502896308899,0.2551196813583374,0.9929869771003723,0.0905260294675827],[0.9707273244857788,0.6956090927124023,0.6280022263526917,0.7902868390083313,0.9343841671943665,0.9895793795585632,0.9436282515525818,-0.10834990441799164,-0.3431110680103302,0.9986709952354431,0.508673906326294,0.9949509501457214],[0.8283132910728455,0.9432437419891357,0.9491764903068542,0.9995353817939758,0.5712320804595947,0.8055236339569092,0.6781865954399109,0.8272573351860046,0.8314797282218933,0.8778655529022217,0.9944959282875061,0.997386634349823]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0,\"cmin\":-1.0,\"cmax\":1.0},\"title\":{\"text\":\"OV Copying Score for each head in GPT-2 Small\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"039423ef-6937-4f4c-a9b7-5f7f1ec8d99c\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"039423ef-6937-4f4c-a9b7-5f7f1ec8d99c\")) {                    Plotly.newPlot(                        \"039423ef-6937-4f4c-a9b7-5f7f1ec8d99c\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.8356367349624634,0.5853534936904907,0.5105841159820557,0.7843377590179443,0.8644161224365234,0.7026589512825012,0.8969924449920654,0.5868823528289795,0.42486509680747986,-0.16337518393993378,0.46268585324287415,0.276053786277771],[-0.052920110523700714,-0.31773144006729126,-0.4810579717159271,-0.7838066220283508,-0.6360211968421936,-0.77586829662323,0.9681803584098816,0.8119116425514221,-0.7510464787483215,-0.6878445744514465,-0.6429887413978577,-0.9985856413841248],[-0.6598325967788696,0.9152501821517944,0.5461500883102417,-0.48743969202041626,0.7720563411712646,-0.7541061639785767,-0.8472450971603394,-0.6948987245559692,-0.1557510495185852,0.2444225549697876,-0.9106622338294983,-0.9439151287078857],[0.648689866065979,-0.5592910647392273,0.5935593247413635,0.7843040823936462,-0.8150346279144287,0.613004744052887,0.16785882413387299,0.35195910930633545,-0.6837262511253357,0.2223764955997467,-0.9929219484329224,0.6535818576812744],[0.5740953087806702,0.3640134036540985,0.09609051048755646,0.9359624981880188,-0.9228776097297668,0.6191076636314392,-0.33572638034820557,-0.998464822769165,0.6448632478713989,0.8468661308288574,-0.7557656764984131,0.9527971744537354],[0.7326544523239136,0.5324169397354126,0.9732670187950134,0.7239246964454651,0.2553895115852356,0.8158416152000427,0.665579080581665,0.9287100434303284,-0.5660436153411865,-0.8908745050430298,0.9834234118461609,-0.9981179237365723],[0.9698692560195923,0.7439671754837036,-0.35639333724975586,0.6022987365722656,0.9708116054534912,-0.9278275966644287,-0.9962316155433655,0.834520697593689,0.9714328050613403,0.8158544898033142,0.5902575850486755,0.8199343681335449],[0.9820227026939392,0.9859329462051392,0.5152460932731628,-0.5610517263412476,0.9663666486740112,0.9495157599449158,-0.5204814076423645,0.3104752004146576,0.9859083890914917,0.7797460556030273,0.6738530397415161,0.39197394251823425],[-0.906204104423523,0.11750960350036621,0.8077874779701233,0.416930615901947,0.9829014539718628,0.9902300834655762,0.7847104668617249,0.9945629835128784,0.9868024587631226,-0.26804426312446594,-0.9908867478370667,0.745792806148529],[0.9906190633773804,-0.18231146037578583,0.9757838249206543,0.9986750483512878,0.2544330954551697,-0.954406201839447,0.586924135684967,-0.23537978529930115,0.9550502896308899,0.2551196813583374,0.9929869771003723,0.0905260294675827],[0.9707273244857788,0.6956090927124023,0.6280022263526917,0.7902868390083313,0.9343841671943665,0.9895793795585632,0.9436282515525818,-0.10834990441799164,-0.3431110680103302,0.9986709952354431,0.508673906326294,0.9949509501457214],[0.8283132910728455,0.9432437419891357,0.9491764903068542,0.9995353817939758,0.5712320804595947,0.8055236339569092,0.6781865954399109,0.8272573351860046,0.8314797282218933,0.8778655529022217,0.9944959282875061,0.997386634349823]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0,\"cmin\":-1.0,\"cmax\":1.0},\"title\":{\"text\":\"OV Copying Score for each head in GPT-2 Small\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('e3552224-0615-4466-b566-225b4371075b');\n",
+       "var gd = document.getElementById('039423ef-6937-4f4c-a9b7-5f7f1ec8d99c');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -2736,10 +2729,10 @@
    "execution_count": 51,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:42:33.324543Z",
-     "iopub.status.busy": "2024-09-05T17:42:33.324195Z",
-     "iopub.status.idle": "2024-09-05T17:42:33.354764Z",
-     "shell.execute_reply": "2024-09-05T17:42:33.354210Z"
+     "iopub.execute_input": "2024-09-10T17:16:08.509550Z",
+     "iopub.status.busy": "2024-09-10T17:16:08.509375Z",
+     "iopub.status.idle": "2024-09-10T17:16:08.540482Z",
+     "shell.execute_reply": "2024-09-10T17:16:08.539989Z"
     }
    },
    "outputs": [
@@ -2750,9 +2743,9 @@
        "<head><meta charset=\"utf-8\" /></head>\n",
        "<body>\n",
        "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
-       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"2be56e5c-d2b2-481c-94a5-bddccdf1e7cd\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"2be56e5c-d2b2-481c-94a5-bddccdf1e7cd\")) {                    Plotly.newPlot(                        \"2be56e5c-d2b2-481c-94a5-bddccdf1e7cd\",                        [{\"hovertemplate\":\"\\u003cb\\u003e%{hovertext}\\u003c\\u002fb\\u003e\\u003cbr\\u003e\\u003cbr\\u003eFull OV Copying Score=%{x}\\u003cbr\\u003eOV Copying Score=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"hovertext\":[\"L0H0\",\"L0H1\",\"L0H2\",\"L0H3\",\"L0H4\",\"L0H5\",\"L0H6\",\"L0H7\",\"L0H8\",\"L0H9\",\"L0H10\",\"L0H11\",\"L1H0\",\"L1H1\",\"L1H2\",\"L1H3\",\"L1H4\",\"L1H5\",\"L1H6\",\"L1H7\",\"L1H8\",\"L1H9\",\"L1H10\",\"L1H11\",\"L2H0\",\"L2H1\",\"L2H2\",\"L2H3\",\"L2H4\",\"L2H5\",\"L2H6\",\"L2H7\",\"L2H8\",\"L2H9\",\"L2H10\",\"L2H11\",\"L3H0\",\"L3H1\",\"L3H2\",\"L3H3\",\"L3H4\",\"L3H5\",\"L3H6\",\"L3H7\",\"L3H8\",\"L3H9\",\"L3H10\",\"L3H11\",\"L4H0\",\"L4H1\",\"L4H2\",\"L4H3\",\"L4H4\",\"L4H5\",\"L4H6\",\"L4H7\",\"L4H8\",\"L4H9\",\"L4H10\",\"L4H11\",\"L5H0\",\"L5H1\",\"L5H2\",\"L5H3\",\"L5H4\",\"L5H5\",\"L5H6\",\"L5H7\",\"L5H8\",\"L5H9\",\"L5H10\",\"L5H11\",\"L6H0\",\"L6H1\",\"L6H2\",\"L6H3\",\"L6H4\",\"L6H5\",\"L6H6\",\"L6H7\",\"L6H8\",\"L6H9\",\"L6H10\",\"L6H11\",\"L7H0\",\"L7H1\",\"L7H2\",\"L7H3\",\"L7H4\",\"L7H5\",\"L7H6\",\"L7H7\",\"L7H8\",\"L7H9\",\"L7H10\",\"L7H11\",\"L8H0\",\"L8H1\",\"L8H2\",\"L8H3\",\"L8H4\",\"L8H5\",\"L8H6\",\"L8H7\",\"L8H8\",\"L8H9\",\"L8H10\",\"L8H11\",\"L9H0\",\"L9H1\",\"L9H2\",\"L9H3\",\"L9H4\",\"L9H5\",\"L9H6\",\"L9H7\",\"L9H8\",\"L9H9\",\"L9H10\",\"L9H11\",\"L10H0\",\"L10H1\",\"L10H2\",\"L10H3\",\"L10H4\",\"L10H5\",\"L10H6\",\"L10H7\",\"L10H8\",\"L10H9\",\"L10H10\",\"L10H11\",\"L11H0\",\"L11H1\",\"L11H2\",\"L11H3\",\"L11H4\",\"L11H5\",\"L11H6\",\"L11H7\",\"L11H8\",\"L11H9\",\"L11H10\",\"L11H11\"],\"legendgroup\":\"\",\"marker\":{\"color\":\"#636efa\",\"symbol\":\"circle\"},\"mode\":\"markers\",\"name\":\"\",\"orientation\":\"v\",\"showlegend\":false,\"x\":[0.8356367349624634,0.5853534936904907,0.5105841159820557,0.7843377590179443,0.8644161224365234,0.7026589512825012,0.8969924449920654,0.5868823528289795,0.42486509680747986,-0.16337518393993378,0.46268585324287415,0.276053786277771,-0.052920110523700714,-0.31773144006729126,-0.4810579717159271,-0.7838066220283508,-0.6360211968421936,-0.77586829662323,0.9681803584098816,0.8119116425514221,-0.7510464787483215,-0.6878445744514465,-0.6429887413978577,-0.9985856413841248,-0.6598325967788696,0.9152501821517944,0.5461500883102417,-0.48743969202041626,0.7720563411712646,-0.7541061639785767,-0.8472450971603394,-0.6948987245559692,-0.1557510495185852,0.2444225549697876,-0.9106622338294983,-0.9439151287078857,0.648689866065979,-0.5592910647392273,0.5935593247413635,0.7843040823936462,-0.8150346279144287,0.613004744052887,0.16785882413387299,0.35195910930633545,-0.6837262511253357,0.2223764955997467,-0.9929219484329224,0.6535818576812744,0.5740953087806702,0.3640134036540985,0.09609051048755646,0.9359624981880188,-0.9228776097297668,0.6191076636314392,-0.33572638034820557,-0.998464822769165,0.6448632478713989,0.8468661308288574,-0.7557656764984131,0.9527971744537354,0.7326544523239136,0.5324169397354126,0.9732670187950134,0.7239246964454651,0.2553895115852356,0.8158416152000427,0.665579080581665,0.9287100434303284,-0.5660436153411865,-0.8908745050430298,0.9834234118461609,-0.9981179237365723,0.9698692560195923,0.7439671754837036,-0.35639333724975586,0.6022987365722656,0.9708116054534912,-0.9278275966644287,-0.9962316155433655,0.834520697593689,0.9714328050613403,0.8158544898033142,0.5902575850486755,0.8199343681335449,0.9820227026939392,0.9859329462051392,0.5152460932731628,-0.5610517263412476,0.9663666486740112,0.9495157599449158,-0.5204814076423645,0.3104752004146576,0.9859083890914917,0.7797460556030273,0.6738530397415161,0.39197394251823425,-0.906204104423523,0.11750960350036621,0.8077874779701233,0.416930615901947,0.9829014539718628,0.9902300834655762,0.7847104668617249,0.9945629835128784,0.9868024587631226,-0.26804426312446594,-0.9908867478370667,0.745792806148529,0.9906190633773804,-0.18231146037578583,0.9757838249206543,0.9986750483512878,0.2544330954551697,-0.954406201839447,0.586924135684967,-0.23537978529930115,0.9550502896308899,0.2551196813583374,0.9929869771003723,0.0905260294675827,0.9707273244857788,0.6956090927124023,0.6280022263526917,0.7902868390083313,0.9343841671943665,0.9895793795585632,0.9436282515525818,-0.10834990441799164,-0.3431110680103302,0.9986709952354431,0.508673906326294,0.9949509501457214,0.8283132910728455,0.9432437419891357,0.9491764903068542,0.9995353817939758,0.5712320804595947,0.8055236339569092,0.6781865954399109,0.8272573351860046,0.8314797282218933,0.8778655529022217,0.9944959282875061,0.997386634349823],\"xaxis\":\"x\",\"y\":[0.7775008678436279,0.35272663831710815,0.25961872935295105,0.6670258045196533,0.838425874710083,0.5584433078765869,0.844474732875824,0.41379114985466003,0.244889497756958,0.02815753035247326,0.3584096431732178,0.16288283467292786,-0.4541913866996765,-0.6529324650764465,-0.5484569668769836,-0.7990368604660034,-0.7736426591873169,-0.8522582650184631,0.977432370185852,0.6626251935958862,-0.7303223609924316,-0.7007019519805908,-0.6946623921394348,-0.9996723532676697,-0.7837162613868713,0.8967758417129517,0.4750956892967224,-0.6671973466873169,0.7881463170051575,-0.8547748923301697,-0.9054183959960938,-0.5749384760856628,-0.321751207113266,-0.0285941194742918,-0.9247617125511169,-0.9699268937110901,0.5864036083221436,-0.76143479347229,0.5971695184707642,0.7854391932487488,-0.8788884878158569,0.3908745348453522,0.0447387658059597,0.1102800965309143,-0.8169987201690674,0.2212953418493271,-0.9939578771591187,0.5774401426315308,0.5254791378974915,0.30490121245384216,-0.10729170590639114,0.9433152675628662,-0.9314427971839905,0.5273631811141968,-0.4264712929725647,-0.9984429478645325,0.5296757817268372,0.8604294657707214,-0.8895052075386047,0.9556970000267029,0.6629188656806946,0.4295697808265686,0.9736858010292053,0.655548095703125,0.12201863527297974,0.7442769408226013,0.5037954449653625,0.95253586769104,-0.6507166624069214,-0.9316278100013733,0.9791510701179504,-0.9972586035728455,0.9613031148910522,0.7501779794692993,-0.3806658983230591,0.6429785490036011,0.9557769298553467,-0.9428837895393372,-0.9948079586029053,0.7852989435195923,0.9657301306724548,0.7073014974594116,0.36872273683547974,0.8128010034561157,0.9659482836723328,0.9730120301246643,0.31900620460510254,-0.30290529131889343,0.9790952801704407,0.9357922673225403,-0.5550314784049988,-0.0054661668837070465,0.986777663230896,0.8249568343162537,0.5664296746253967,0.1000528484582901,-0.9464486837387085,-0.25471991300582886,0.6522327065467834,0.14152583479881287,0.9884141683578491,0.9860584735870361,0.6949271559715271,0.9901811480522156,0.9791203141212463,-0.23595543205738068,-0.982071042060852,0.6506688594818115,0.9895945191383362,-0.291781485080719,0.9714024662971497,0.9951602220535278,0.18783727288246155,-0.9460937976837158,0.4780191481113434,-0.2489192634820938,0.9437099099159241,0.11866225302219391,0.9941242933273315,-0.38088199496269226,0.9564487338066101,0.5542722344398499,0.42118069529533386,0.6628788113594055,0.8659593462944031,0.9937117695808411,0.9069075584411621,0.3981107473373413,-0.4134218096733093,0.9971914887428284,0.3459664583206177,0.9938657283782959,0.5891268253326416,0.9313738942146301,0.9268401861190796,0.9993563890457153,0.6227542161941528,0.8463947772979736,0.6584343910217285,0.8423123955726624,0.2978499233722687,0.8728678822517395,0.9963143467903137,0.9867526292800903],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Full OV Copying Score\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"OV Copying Score\"}},\"legend\":{\"tracegroupgap\":0},\"title\":{\"text\":\"OV Copying Score for each head in GPT-2 Small\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"2386d5b5-f55c-48a9-850d-e7c0f3dfdb0b\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"2386d5b5-f55c-48a9-850d-e7c0f3dfdb0b\")) {                    Plotly.newPlot(                        \"2386d5b5-f55c-48a9-850d-e7c0f3dfdb0b\",                        [{\"hovertemplate\":\"\\u003cb\\u003e%{hovertext}\\u003c\\u002fb\\u003e\\u003cbr\\u003e\\u003cbr\\u003eFull OV Copying Score=%{x}\\u003cbr\\u003eOV Copying Score=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"hovertext\":[\"L0H0\",\"L0H1\",\"L0H2\",\"L0H3\",\"L0H4\",\"L0H5\",\"L0H6\",\"L0H7\",\"L0H8\",\"L0H9\",\"L0H10\",\"L0H11\",\"L1H0\",\"L1H1\",\"L1H2\",\"L1H3\",\"L1H4\",\"L1H5\",\"L1H6\",\"L1H7\",\"L1H8\",\"L1H9\",\"L1H10\",\"L1H11\",\"L2H0\",\"L2H1\",\"L2H2\",\"L2H3\",\"L2H4\",\"L2H5\",\"L2H6\",\"L2H7\",\"L2H8\",\"L2H9\",\"L2H10\",\"L2H11\",\"L3H0\",\"L3H1\",\"L3H2\",\"L3H3\",\"L3H4\",\"L3H5\",\"L3H6\",\"L3H7\",\"L3H8\",\"L3H9\",\"L3H10\",\"L3H11\",\"L4H0\",\"L4H1\",\"L4H2\",\"L4H3\",\"L4H4\",\"L4H5\",\"L4H6\",\"L4H7\",\"L4H8\",\"L4H9\",\"L4H10\",\"L4H11\",\"L5H0\",\"L5H1\",\"L5H2\",\"L5H3\",\"L5H4\",\"L5H5\",\"L5H6\",\"L5H7\",\"L5H8\",\"L5H9\",\"L5H10\",\"L5H11\",\"L6H0\",\"L6H1\",\"L6H2\",\"L6H3\",\"L6H4\",\"L6H5\",\"L6H6\",\"L6H7\",\"L6H8\",\"L6H9\",\"L6H10\",\"L6H11\",\"L7H0\",\"L7H1\",\"L7H2\",\"L7H3\",\"L7H4\",\"L7H5\",\"L7H6\",\"L7H7\",\"L7H8\",\"L7H9\",\"L7H10\",\"L7H11\",\"L8H0\",\"L8H1\",\"L8H2\",\"L8H3\",\"L8H4\",\"L8H5\",\"L8H6\",\"L8H7\",\"L8H8\",\"L8H9\",\"L8H10\",\"L8H11\",\"L9H0\",\"L9H1\",\"L9H2\",\"L9H3\",\"L9H4\",\"L9H5\",\"L9H6\",\"L9H7\",\"L9H8\",\"L9H9\",\"L9H10\",\"L9H11\",\"L10H0\",\"L10H1\",\"L10H2\",\"L10H3\",\"L10H4\",\"L10H5\",\"L10H6\",\"L10H7\",\"L10H8\",\"L10H9\",\"L10H10\",\"L10H11\",\"L11H0\",\"L11H1\",\"L11H2\",\"L11H3\",\"L11H4\",\"L11H5\",\"L11H6\",\"L11H7\",\"L11H8\",\"L11H9\",\"L11H10\",\"L11H11\"],\"legendgroup\":\"\",\"marker\":{\"color\":\"#636efa\",\"symbol\":\"circle\"},\"mode\":\"markers\",\"name\":\"\",\"orientation\":\"v\",\"showlegend\":false,\"x\":[0.8356367349624634,0.5853534936904907,0.5105841159820557,0.7843377590179443,0.8644161224365234,0.7026589512825012,0.8969924449920654,0.5868823528289795,0.42486509680747986,-0.16337518393993378,0.46268585324287415,0.276053786277771,-0.052920110523700714,-0.31773144006729126,-0.4810579717159271,-0.7838066220283508,-0.6360211968421936,-0.77586829662323,0.9681803584098816,0.8119116425514221,-0.7510464787483215,-0.6878445744514465,-0.6429887413978577,-0.9985856413841248,-0.6598325967788696,0.9152501821517944,0.5461500883102417,-0.48743969202041626,0.7720563411712646,-0.7541061639785767,-0.8472450971603394,-0.6948987245559692,-0.1557510495185852,0.2444225549697876,-0.9106622338294983,-0.9439151287078857,0.648689866065979,-0.5592910647392273,0.5935593247413635,0.7843040823936462,-0.8150346279144287,0.613004744052887,0.16785882413387299,0.35195910930633545,-0.6837262511253357,0.2223764955997467,-0.9929219484329224,0.6535818576812744,0.5740953087806702,0.3640134036540985,0.09609051048755646,0.9359624981880188,-0.9228776097297668,0.6191076636314392,-0.33572638034820557,-0.998464822769165,0.6448632478713989,0.8468661308288574,-0.7557656764984131,0.9527971744537354,0.7326544523239136,0.5324169397354126,0.9732670187950134,0.7239246964454651,0.2553895115852356,0.8158416152000427,0.665579080581665,0.9287100434303284,-0.5660436153411865,-0.8908745050430298,0.9834234118461609,-0.9981179237365723,0.9698692560195923,0.7439671754837036,-0.35639333724975586,0.6022987365722656,0.9708116054534912,-0.9278275966644287,-0.9962316155433655,0.834520697593689,0.9714328050613403,0.8158544898033142,0.5902575850486755,0.8199343681335449,0.9820227026939392,0.9859329462051392,0.5152460932731628,-0.5610517263412476,0.9663666486740112,0.9495157599449158,-0.5204814076423645,0.3104752004146576,0.9859083890914917,0.7797460556030273,0.6738530397415161,0.39197394251823425,-0.906204104423523,0.11750960350036621,0.8077874779701233,0.416930615901947,0.9829014539718628,0.9902300834655762,0.7847104668617249,0.9945629835128784,0.9868024587631226,-0.26804426312446594,-0.9908867478370667,0.745792806148529,0.9906190633773804,-0.18231146037578583,0.9757838249206543,0.9986750483512878,0.2544330954551697,-0.954406201839447,0.586924135684967,-0.23537978529930115,0.9550502896308899,0.2551196813583374,0.9929869771003723,0.0905260294675827,0.9707273244857788,0.6956090927124023,0.6280022263526917,0.7902868390083313,0.9343841671943665,0.9895793795585632,0.9436282515525818,-0.10834990441799164,-0.3431110680103302,0.9986709952354431,0.508673906326294,0.9949509501457214,0.8283132910728455,0.9432437419891357,0.9491764903068542,0.9995353817939758,0.5712320804595947,0.8055236339569092,0.6781865954399109,0.8272573351860046,0.8314797282218933,0.8778655529022217,0.9944959282875061,0.997386634349823],\"xaxis\":\"x\",\"y\":[0.7775008678436279,0.35272663831710815,0.25961872935295105,0.6670258045196533,0.838425874710083,0.5584433078765869,0.844474732875824,0.41379114985466003,0.244889497756958,0.02815753035247326,0.3584096431732178,0.16288283467292786,-0.4541913866996765,-0.6529324650764465,-0.5484569668769836,-0.7990368604660034,-0.7736426591873169,-0.8522582650184631,0.977432370185852,0.6626251935958862,-0.7303223609924316,-0.7007019519805908,-0.6946623921394348,-0.9996723532676697,-0.7837162613868713,0.8967758417129517,0.4750956892967224,-0.6671973466873169,0.7881463170051575,-0.8547748923301697,-0.9054183959960938,-0.5749384760856628,-0.321751207113266,-0.0285941194742918,-0.9247617125511169,-0.9699268937110901,0.5864036083221436,-0.76143479347229,0.5971695184707642,0.7854391932487488,-0.8788884878158569,0.3908745348453522,0.0447387658059597,0.1102800965309143,-0.8169987201690674,0.2212953418493271,-0.9939578771591187,0.5774401426315308,0.5254791378974915,0.30490121245384216,-0.10729170590639114,0.9433152675628662,-0.9314427971839905,0.5273631811141968,-0.4264712929725647,-0.9984429478645325,0.5296757817268372,0.8604294657707214,-0.8895052075386047,0.9556970000267029,0.6629188656806946,0.4295697808265686,0.9736858010292053,0.655548095703125,0.12201863527297974,0.7442769408226013,0.5037954449653625,0.95253586769104,-0.6507166624069214,-0.9316278100013733,0.9791510701179504,-0.9972586035728455,0.9613031148910522,0.7501779794692993,-0.3806658983230591,0.6429785490036011,0.9557769298553467,-0.9428837895393372,-0.9948079586029053,0.7852989435195923,0.9657301306724548,0.7073014974594116,0.36872273683547974,0.8128010034561157,0.9659482836723328,0.9730120301246643,0.31900620460510254,-0.30290529131889343,0.9790952801704407,0.9357922673225403,-0.5550314784049988,-0.0054661668837070465,0.986777663230896,0.8249568343162537,0.5664296746253967,0.1000528484582901,-0.9464486837387085,-0.25471991300582886,0.6522327065467834,0.14152583479881287,0.9884141683578491,0.9860584735870361,0.6949271559715271,0.9901811480522156,0.9791203141212463,-0.23595543205738068,-0.982071042060852,0.6506688594818115,0.9895945191383362,-0.291781485080719,0.9714024662971497,0.9951602220535278,0.18783727288246155,-0.9460937976837158,0.4780191481113434,-0.2489192634820938,0.9437099099159241,0.11866225302219391,0.9941242933273315,-0.38088199496269226,0.9564487338066101,0.5542722344398499,0.42118069529533386,0.6628788113594055,0.8659593462944031,0.9937117695808411,0.9069075584411621,0.3981107473373413,-0.4134218096733093,0.9971914887428284,0.3459664583206177,0.9938657283782959,0.5891268253326416,0.9313738942146301,0.9268401861190796,0.9993563890457153,0.6227542161941528,0.8463947772979736,0.6584343910217285,0.8423123955726624,0.2978499233722687,0.8728678822517395,0.9963143467903137,0.9867526292800903],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Full OV Copying Score\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"OV Copying Score\"}},\"legend\":{\"tracegroupgap\":0},\"title\":{\"text\":\"OV Copying Score for each head in GPT-2 Small\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('2be56e5c-d2b2-481c-94a5-bddccdf1e7cd');\n",
+       "var gd = document.getElementById('2386d5b5-f55c-48a9-850d-e7c0f3dfdb0b');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -2792,10 +2785,10 @@
    "execution_count": 52,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:42:33.356956Z",
-     "iopub.status.busy": "2024-09-05T17:42:33.356612Z",
-     "iopub.status.idle": "2024-09-05T17:42:33.360567Z",
-     "shell.execute_reply": "2024-09-05T17:42:33.359969Z"
+     "iopub.execute_input": "2024-09-10T17:16:08.542483Z",
+     "iopub.status.busy": "2024-09-10T17:16:08.542310Z",
+     "iopub.status.idle": "2024-09-10T17:16:08.546050Z",
+     "shell.execute_reply": "2024-09-10T17:16:08.545593Z"
     }
    },
    "outputs": [
@@ -2836,17 +2829,17 @@
    "execution_count": 53,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:42:33.362701Z",
-     "iopub.status.busy": "2024-09-05T17:42:33.362284Z",
-     "iopub.status.idle": "2024-09-05T17:42:35.695082Z",
-     "shell.execute_reply": "2024-09-05T17:42:35.694449Z"
+     "iopub.execute_input": "2024-09-10T17:16:08.548236Z",
+     "iopub.status.busy": "2024-09-10T17:16:08.547915Z",
+     "iopub.status.idle": "2024-09-10T17:16:10.907401Z",
+     "shell.execute_reply": "2024-09-10T17:16:10.906818Z"
     }
    },
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "f1b89201b64941828ea8f7bb5ebac2ca",
+       "model_id": "008a6d643cb74e9da094f214bdade5b2",
        "version_major": 2,
        "version_minor": 0
       },
@@ -2922,10 +2915,10 @@
    "execution_count": 54,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:42:35.697604Z",
-     "iopub.status.busy": "2024-09-05T17:42:35.697211Z",
-     "iopub.status.idle": "2024-09-05T17:42:35.702722Z",
-     "shell.execute_reply": "2024-09-05T17:42:35.702242Z"
+     "iopub.execute_input": "2024-09-10T17:16:10.909860Z",
+     "iopub.status.busy": "2024-09-10T17:16:10.909515Z",
+     "iopub.status.idle": "2024-09-10T17:16:10.915015Z",
+     "shell.execute_reply": "2024-09-10T17:16:10.914544Z"
     }
    },
    "outputs": [],
@@ -2987,10 +2980,10 @@
    "execution_count": 55,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:42:35.704735Z",
-     "iopub.status.busy": "2024-09-05T17:42:35.704412Z",
-     "iopub.status.idle": "2024-09-05T17:42:35.708665Z",
-     "shell.execute_reply": "2024-09-05T17:42:35.708205Z"
+     "iopub.execute_input": "2024-09-10T17:16:10.916891Z",
+     "iopub.status.busy": "2024-09-10T17:16:10.916726Z",
+     "iopub.status.idle": "2024-09-10T17:16:10.920838Z",
+     "shell.execute_reply": "2024-09-10T17:16:10.920292Z"
     }
    },
    "outputs": [
@@ -3029,10 +3022,10 @@
    "execution_count": 56,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:42:35.710788Z",
-     "iopub.status.busy": "2024-09-05T17:42:35.710411Z",
-     "iopub.status.idle": "2024-09-05T17:42:35.714593Z",
-     "shell.execute_reply": "2024-09-05T17:42:35.713989Z"
+     "iopub.execute_input": "2024-09-10T17:16:10.922650Z",
+     "iopub.status.busy": "2024-09-10T17:16:10.922488Z",
+     "iopub.status.idle": "2024-09-10T17:16:10.926462Z",
+     "shell.execute_reply": "2024-09-10T17:16:10.925896Z"
     }
    },
    "outputs": [
@@ -3103,10 +3096,10 @@
    "execution_count": 57,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:42:35.716820Z",
-     "iopub.status.busy": "2024-09-05T17:42:35.716483Z",
-     "iopub.status.idle": "2024-09-05T17:42:36.351090Z",
-     "shell.execute_reply": "2024-09-05T17:42:36.350479Z"
+     "iopub.execute_input": "2024-09-10T17:16:10.928457Z",
+     "iopub.status.busy": "2024-09-10T17:16:10.928135Z",
+     "iopub.status.idle": "2024-09-10T17:16:11.193571Z",
+     "shell.execute_reply": "2024-09-10T17:16:11.192992Z"
     }
    },
    "outputs": [
@@ -3117,9 +3110,9 @@
        "<head><meta charset=\"utf-8\" /></head>\n",
        "<body>\n",
        "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
-       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"2f4b0396-49d9-4cde-b98b-91ebcc4cd205\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"2f4b0396-49d9-4cde-b98b-91ebcc4cd205\")) {                    Plotly.newPlot(                        \"2f4b0396-49d9-4cde-b98b-91ebcc4cd205\",                        [{\"hovertemplate\":\"variable=0\\u003cbr\\u003eindex=%{x}\\u003cbr\\u003evalue=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"0\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines+markers\",\"name\":\"0\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162],\"xaxis\":\"x\",\"y\":[262144,2621440,4718592,7077888,9175040,11272192,13631488,15728640,18087936,20185088,22282240,33292288,44302336,55312384,66322432,77332480,88342528,99352576,110362624,121372672,132382720,143392768,154402816,165412864,176422912,187432960,198443008,209453056,220463104,264503296,308281344,352321536,396361728,440401920,484442112,528482304,572522496,616300544,660340736,704380928,748421120,792461312,836501504,880279552,924319744,968359936,1012400128,1056440320,1100480512,1144520704,1188298752,1232338944,1276379136,1320419328,1364459520,1408499712,1452277760,1496317952,1540358144,1584398336,1628438528,1672478720,1716518912,1760296960,1804337152,1848377344,1892417536,1936457728,1980497920,2024275968,2068316160,2112356352,2156396544,2200436736,2420375552,2640314368,2860515328,3080454144,3300392960,3520331776,3740270592,3960471552,4180410368,4400349184,4620288000,4840488960,5060427776,5280366592,5500305408,5720506368,5940445184,6160384000,6380322816,6600523776,6820462592,7040401408,7260340224,7480279040,7700480000,7920418816,8140357632,8360296448,8580497408,8800436224,9020375040,9240313856,9460514816,9680453632,9900392448,10120331264,10340270080,10560471040,10780409856,11000348672,11220287488,11440488448,11660427264,11880366080,12100304896,12320505856,12540444672,12760383488,12980322304,13200523264,13420462080,13640400896,13860339712,14080278528,14300479488,14520418304,14740357120,14960295936,15180496896,15400435712,15620374528,15840313344,16060514304,16280453120,16500391936,16720330752,16940269568,17160470528,17380409344,17600348160,17820286976,18040487936,18260426752,18480365568,18700304384,18920505344,19140444160,19360382976,19580321792,19800522752,20020461568,20240400384,20460339200,20680278016,20900478976,21120417792,21340356608,21560295424,21780496384],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"index\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"value\"},\"type\":\"log\"},\"legend\":{\"title\":{\"text\":\"variable\"},\"tracegroupgap\":0},\"title\":{\"text\":\"Checkpoint Values for attn-only-2l (Log scale)\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"4602c771-fd44-40e0-a92b-9fc2cdaf024a\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"4602c771-fd44-40e0-a92b-9fc2cdaf024a\")) {                    Plotly.newPlot(                        \"4602c771-fd44-40e0-a92b-9fc2cdaf024a\",                        [{\"hovertemplate\":\"variable=0\\u003cbr\\u003eindex=%{x}\\u003cbr\\u003evalue=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"0\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines+markers\",\"name\":\"0\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162],\"xaxis\":\"x\",\"y\":[262144,2621440,4718592,7077888,9175040,11272192,13631488,15728640,18087936,20185088,22282240,33292288,44302336,55312384,66322432,77332480,88342528,99352576,110362624,121372672,132382720,143392768,154402816,165412864,176422912,187432960,198443008,209453056,220463104,264503296,308281344,352321536,396361728,440401920,484442112,528482304,572522496,616300544,660340736,704380928,748421120,792461312,836501504,880279552,924319744,968359936,1012400128,1056440320,1100480512,1144520704,1188298752,1232338944,1276379136,1320419328,1364459520,1408499712,1452277760,1496317952,1540358144,1584398336,1628438528,1672478720,1716518912,1760296960,1804337152,1848377344,1892417536,1936457728,1980497920,2024275968,2068316160,2112356352,2156396544,2200436736,2420375552,2640314368,2860515328,3080454144,3300392960,3520331776,3740270592,3960471552,4180410368,4400349184,4620288000,4840488960,5060427776,5280366592,5500305408,5720506368,5940445184,6160384000,6380322816,6600523776,6820462592,7040401408,7260340224,7480279040,7700480000,7920418816,8140357632,8360296448,8580497408,8800436224,9020375040,9240313856,9460514816,9680453632,9900392448,10120331264,10340270080,10560471040,10780409856,11000348672,11220287488,11440488448,11660427264,11880366080,12100304896,12320505856,12540444672,12760383488,12980322304,13200523264,13420462080,13640400896,13860339712,14080278528,14300479488,14520418304,14740357120,14960295936,15180496896,15400435712,15620374528,15840313344,16060514304,16280453120,16500391936,16720330752,16940269568,17160470528,17380409344,17600348160,17820286976,18040487936,18260426752,18480365568,18700304384,18920505344,19140444160,19360382976,19580321792,19800522752,20020461568,20240400384,20460339200,20680278016,20900478976,21120417792,21340356608,21560295424,21780496384],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"index\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"value\"},\"type\":\"log\"},\"legend\":{\"title\":{\"text\":\"variable\"},\"tracegroupgap\":0},\"title\":{\"text\":\"Checkpoint Values for attn-only-2l (Log scale)\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('2f4b0396-49d9-4cde-b98b-91ebcc4cd205');\n",
+       "var gd = document.getElementById('4602c771-fd44-40e0-a92b-9fc2cdaf024a');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -3156,9 +3149,9 @@
        "<head><meta charset=\"utf-8\" /></head>\n",
        "<body>\n",
        "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
-       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"f50d5618-52f9-42e8-a118-5e9a1a641854\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"f50d5618-52f9-42e8-a118-5e9a1a641854\")) {                    Plotly.newPlot(                        \"f50d5618-52f9-42e8-a118-5e9a1a641854\",                        [{\"hovertemplate\":\"variable=0\\u003cbr\\u003eindex=%{x}\\u003cbr\\u003evalue=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"0\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines+markers\",\"name\":\"0\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162],\"xaxis\":\"x\",\"y\":[196608,3342336,6291456,9240576,12386304,15335424,18284544,21233664,24379392,27328512,30277632,45219840,60358656,75300864,90243072,105381888,120324096,135266304,150208512,165347328,180289536,195231744,210370560,225312768,240254976,255197184,270336000,285278208,300220416,360382464,420347904,480313344,540278784,600244224,660209664,720371712,780337152,840302592,900268032,960233472,1020198912,1080360960,1140326400,1200291840,1260257280,1320222720,1380384768,1440350208,1500315648,1560281088,1620246528,1680211968,1740374016,1800339456,1860304896,1920270336,1980235776,2040201216,2100363264,2160328704,2220294144,2280259584,2340225024,2400387072,2460352512,2520317952,2580283392,2640248832,2700214272,2760376320,2820341760,2880307200,2940272640,3000238080,3300261888,3600285696,3900309504,4200333312,4500357120,4800380928,5100208128,5400231936,5700255744,6000279552,6300303360,6600327168,6900350976,7200374784,7500201984,7800225792,8100249600,8400273408,8700297216,9000321024,9300344832,9600368640,9900392448,10200219648,10500243456,10800267264,11100291072,11400314880,11700338688,12000362496,12300386304,12600213504,12900237312,13200261120,13500284928,13800308736,14100332544,14400356352,14700380160,15000207360,15300231168,15600254976,15900278784,16200302592,16500326400,16800350208,17100374016,17400201216,17700225024,18000248832,18300272640,18600296448,18900320256,19200344064,19500367872,19800391680,20100218880,20400242688,20700266496,21000290304,21300314112,21600337920,21900361728,22200385536,22500212736,22800236544,23100260352,23400284160,23700307968,24000331776,24300355584,24600379392,24900206592,25200230400,25500254208,25800278016,26100301824,26400325632,26700349440,27000373248,27300200448,27600224256,27900248064,28200271872,28500295680,28800319488,29100343296,29400367104,29700390912],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"index\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"value\"},\"type\":\"log\"},\"legend\":{\"title\":{\"text\":\"variable\"},\"tracegroupgap\":0},\"title\":{\"text\":\"Checkpoint Values for solu-12l (Log scale)\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"49f0a47b-d076-4975-a8be-48982bf36ec5\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"49f0a47b-d076-4975-a8be-48982bf36ec5\")) {                    Plotly.newPlot(                        \"49f0a47b-d076-4975-a8be-48982bf36ec5\",                        [{\"hovertemplate\":\"variable=0\\u003cbr\\u003eindex=%{x}\\u003cbr\\u003evalue=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"0\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines+markers\",\"name\":\"0\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162],\"xaxis\":\"x\",\"y\":[196608,3342336,6291456,9240576,12386304,15335424,18284544,21233664,24379392,27328512,30277632,45219840,60358656,75300864,90243072,105381888,120324096,135266304,150208512,165347328,180289536,195231744,210370560,225312768,240254976,255197184,270336000,285278208,300220416,360382464,420347904,480313344,540278784,600244224,660209664,720371712,780337152,840302592,900268032,960233472,1020198912,1080360960,1140326400,1200291840,1260257280,1320222720,1380384768,1440350208,1500315648,1560281088,1620246528,1680211968,1740374016,1800339456,1860304896,1920270336,1980235776,2040201216,2100363264,2160328704,2220294144,2280259584,2340225024,2400387072,2460352512,2520317952,2580283392,2640248832,2700214272,2760376320,2820341760,2880307200,2940272640,3000238080,3300261888,3600285696,3900309504,4200333312,4500357120,4800380928,5100208128,5400231936,5700255744,6000279552,6300303360,6600327168,6900350976,7200374784,7500201984,7800225792,8100249600,8400273408,8700297216,9000321024,9300344832,9600368640,9900392448,10200219648,10500243456,10800267264,11100291072,11400314880,11700338688,12000362496,12300386304,12600213504,12900237312,13200261120,13500284928,13800308736,14100332544,14400356352,14700380160,15000207360,15300231168,15600254976,15900278784,16200302592,16500326400,16800350208,17100374016,17400201216,17700225024,18000248832,18300272640,18600296448,18900320256,19200344064,19500367872,19800391680,20100218880,20400242688,20700266496,21000290304,21300314112,21600337920,21900361728,22200385536,22500212736,22800236544,23100260352,23400284160,23700307968,24000331776,24300355584,24600379392,24900206592,25200230400,25500254208,25800278016,26100301824,26400325632,26700349440,27000373248,27300200448,27600224256,27900248064,28200271872,28500295680,28800319488,29100343296,29400367104,29700390912],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"index\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"value\"},\"type\":\"log\"},\"legend\":{\"title\":{\"text\":\"variable\"},\"tracegroupgap\":0},\"title\":{\"text\":\"Checkpoint Values for solu-12l (Log scale)\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('f50d5618-52f9-42e8-a118-5e9a1a641854');\n",
+       "var gd = document.getElementById('49f0a47b-d076-4975-a8be-48982bf36ec5');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -3195,9 +3188,9 @@
        "<head><meta charset=\"utf-8\" /></head>\n",
        "<body>\n",
        "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
-       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"9b48617d-0936-4432-a9e6-abbcbfa6eda0\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"9b48617d-0936-4432-a9e6-abbcbfa6eda0\")) {                    Plotly.newPlot(                        \"9b48617d-0936-4432-a9e6-abbcbfa6eda0\",                        [{\"hovertemplate\":\"variable=0\\u003cbr\\u003eindex=%{x}\\u003cbr\\u003evalue=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"0\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines+markers\",\"name\":\"0\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332,333,334,335,336,337,338,339,340,341,342,343,344,345,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365,366,367,368,369,370,371,372,373,374,375,376,377,378,379,380,381,382,383,384,385,386,387,388,389,390,391,392,393,394,395,396,397,398,399,400,401,402,403,404,405,406,407,408,409,410,411,412,413,414,415,416,417,418,419,420,421,422,423,424,425,426,427,428,429,430,431,432,433,434,435,436,437,438,439,440,441,442,443,444,445,446,447,448,449,450,451,452,453,454,455,456,457,458,459,460,461,462,463,464,465,466,467,468,469,470,471,472,473,474,475,476,477,478,479,480,481,482,483,484,485,486,487,488,489,490,491,492,493,494,495,496,497,498,499,500,501,502,503,504,505,506,507,508,509,510,511,512,513,514,515,516,517,518,519,520,521,522,523,524,525,526,527,528,529,530,531,532,533,534,535,536,537,538,539,540,541,542,543,544,545,546,547,548,549,550,551,552,553,554,555,556,557,558,559,560,561,562,563,564,565,566,567,568,569,570,571,572,573,574,575,576,577,578,579,580,581,582,583,584,585,586,587,588,589,590,591,592,593,594,595,596,597,598,599,600,601,602,603,604,605,606,607,608],\"xaxis\":\"x\",\"y\":[0,10,20,30,40,50,60,70,80,90,100,150,200,250,300,350,400,450,500,550,600,650,700,750,800,850,900,950,1000,1050,1100,1150,1200,1250,1300,1350,1400,1450,1500,1550,1600,1650,1700,1750,1800,1850,1900,1950,2000,2100,2200,2300,2400,2500,2600,2700,2800,2900,3000,3100,3200,3300,3400,3500,3600,3700,3800,3900,4000,4100,4200,4300,4400,4500,4600,4700,4800,4900,5000,5100,5200,5300,5400,5500,5600,5700,5800,5900,6000,6100,6200,6300,6400,6500,6600,6700,6800,6900,7000,7100,7200,7300,7400,7500,7600,7700,7800,7900,8000,8100,8200,8300,8400,8500,8600,8700,8800,8900,9000,9100,9200,9300,9400,9500,9600,9700,9800,9900,10000,10100,10200,10300,10400,10500,10600,10700,10800,10900,11000,11100,11200,11300,11400,11500,11600,11700,11800,11900,12000,12100,12200,12300,12400,12500,12600,12700,12800,12900,13000,13100,13200,13300,13400,13500,13600,13700,13800,13900,14000,14100,14200,14300,14400,14500,14600,14700,14800,14900,15000,15100,15200,15300,15400,15500,15600,15700,15800,15900,16000,16100,16200,16300,16400,16500,16600,16700,16800,16900,17000,17100,17200,17300,17400,17500,17600,17700,17800,17900,18000,18100,18200,18300,18400,18500,18600,18700,18800,18900,19000,19100,19200,19300,19400,19500,19600,19700,19800,19900,20000,21000,22000,23000,24000,25000,26000,27000,28000,29000,30000,31000,32000,33000,34000,35000,36000,37000,38000,39000,40000,41000,42000,43000,44000,45000,46000,47000,48000,49000,50000,51000,52000,53000,54000,55000,56000,57000,58000,59000,60000,61000,62000,63000,64000,65000,66000,67000,68000,69000,70000,71000,72000,73000,74000,75000,76000,77000,78000,79000,80000,81000,82000,83000,84000,85000,86000,87000,88000,89000,90000,91000,92000,93000,94000,95000,96000,97000,98000,99000,100000,101000,102000,103000,104000,105000,106000,107000,108000,109000,110000,111000,112000,113000,114000,115000,116000,117000,118000,119000,120000,121000,122000,123000,124000,125000,126000,127000,128000,129000,130000,131000,132000,133000,134000,135000,136000,137000,138000,139000,140000,141000,142000,143000,144000,145000,146000,147000,148000,149000,150000,151000,152000,153000,154000,155000,156000,157000,158000,159000,160000,161000,162000,163000,164000,165000,166000,167000,168000,169000,170000,171000,172000,173000,174000,175000,176000,177000,178000,179000,180000,181000,182000,183000,184000,185000,186000,187000,188000,189000,190000,191000,192000,193000,194000,195000,196000,197000,198000,199000,200000,201000,202000,203000,204000,205000,206000,207000,208000,209000,210000,211000,212000,213000,214000,215000,216000,217000,218000,219000,220000,221000,222000,223000,224000,225000,226000,227000,228000,229000,230000,231000,232000,233000,234000,235000,236000,237000,238000,239000,240000,241000,242000,243000,244000,245000,246000,247000,248000,249000,250000,251000,252000,253000,254000,255000,256000,257000,258000,259000,260000,261000,262000,263000,264000,265000,266000,267000,268000,269000,270000,271000,272000,273000,274000,275000,276000,277000,278000,279000,280000,281000,282000,283000,284000,285000,286000,287000,288000,289000,290000,291000,292000,293000,294000,295000,296000,297000,298000,299000,300000,301000,302000,303000,304000,305000,306000,307000,308000,309000,310000,311000,312000,313000,314000,315000,316000,317000,318000,319000,320000,321000,322000,323000,324000,325000,326000,327000,328000,329000,330000,331000,332000,333000,334000,335000,336000,337000,338000,339000,340000,341000,342000,343000,344000,345000,346000,347000,348000,349000,350000,351000,352000,353000,354000,355000,356000,357000,358000,359000,360000,361000,362000,363000,364000,365000,366000,367000,368000,369000,370000,371000,372000,373000,374000,375000,376000,377000,378000,379000,380000,381000,382000,383000,384000,385000,386000,387000,388000,389000,390000,391000,392000,393000,394000,395000,396000,397000,398000,399000,400000],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"index\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"value\"},\"type\":\"log\"},\"legend\":{\"title\":{\"text\":\"variable\"},\"tracegroupgap\":0},\"title\":{\"text\":\"Checkpoint Values for stanford-gpt2-small-a (Log scale)\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"922fb711-b683-41df-bedc-350328d71e57\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"922fb711-b683-41df-bedc-350328d71e57\")) {                    Plotly.newPlot(                        \"922fb711-b683-41df-bedc-350328d71e57\",                        [{\"hovertemplate\":\"variable=0\\u003cbr\\u003eindex=%{x}\\u003cbr\\u003evalue=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"0\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines+markers\",\"name\":\"0\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332,333,334,335,336,337,338,339,340,341,342,343,344,345,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365,366,367,368,369,370,371,372,373,374,375,376,377,378,379,380,381,382,383,384,385,386,387,388,389,390,391,392,393,394,395,396,397,398,399,400,401,402,403,404,405,406,407,408,409,410,411,412,413,414,415,416,417,418,419,420,421,422,423,424,425,426,427,428,429,430,431,432,433,434,435,436,437,438,439,440,441,442,443,444,445,446,447,448,449,450,451,452,453,454,455,456,457,458,459,460,461,462,463,464,465,466,467,468,469,470,471,472,473,474,475,476,477,478,479,480,481,482,483,484,485,486,487,488,489,490,491,492,493,494,495,496,497,498,499,500,501,502,503,504,505,506,507,508,509,510,511,512,513,514,515,516,517,518,519,520,521,522,523,524,525,526,527,528,529,530,531,532,533,534,535,536,537,538,539,540,541,542,543,544,545,546,547,548,549,550,551,552,553,554,555,556,557,558,559,560,561,562,563,564,565,566,567,568,569,570,571,572,573,574,575,576,577,578,579,580,581,582,583,584,585,586,587,588,589,590,591,592,593,594,595,596,597,598,599,600,601,602,603,604,605,606,607,608],\"xaxis\":\"x\",\"y\":[0,10,20,30,40,50,60,70,80,90,100,150,200,250,300,350,400,450,500,550,600,650,700,750,800,850,900,950,1000,1050,1100,1150,1200,1250,1300,1350,1400,1450,1500,1550,1600,1650,1700,1750,1800,1850,1900,1950,2000,2100,2200,2300,2400,2500,2600,2700,2800,2900,3000,3100,3200,3300,3400,3500,3600,3700,3800,3900,4000,4100,4200,4300,4400,4500,4600,4700,4800,4900,5000,5100,5200,5300,5400,5500,5600,5700,5800,5900,6000,6100,6200,6300,6400,6500,6600,6700,6800,6900,7000,7100,7200,7300,7400,7500,7600,7700,7800,7900,8000,8100,8200,8300,8400,8500,8600,8700,8800,8900,9000,9100,9200,9300,9400,9500,9600,9700,9800,9900,10000,10100,10200,10300,10400,10500,10600,10700,10800,10900,11000,11100,11200,11300,11400,11500,11600,11700,11800,11900,12000,12100,12200,12300,12400,12500,12600,12700,12800,12900,13000,13100,13200,13300,13400,13500,13600,13700,13800,13900,14000,14100,14200,14300,14400,14500,14600,14700,14800,14900,15000,15100,15200,15300,15400,15500,15600,15700,15800,15900,16000,16100,16200,16300,16400,16500,16600,16700,16800,16900,17000,17100,17200,17300,17400,17500,17600,17700,17800,17900,18000,18100,18200,18300,18400,18500,18600,18700,18800,18900,19000,19100,19200,19300,19400,19500,19600,19700,19800,19900,20000,21000,22000,23000,24000,25000,26000,27000,28000,29000,30000,31000,32000,33000,34000,35000,36000,37000,38000,39000,40000,41000,42000,43000,44000,45000,46000,47000,48000,49000,50000,51000,52000,53000,54000,55000,56000,57000,58000,59000,60000,61000,62000,63000,64000,65000,66000,67000,68000,69000,70000,71000,72000,73000,74000,75000,76000,77000,78000,79000,80000,81000,82000,83000,84000,85000,86000,87000,88000,89000,90000,91000,92000,93000,94000,95000,96000,97000,98000,99000,100000,101000,102000,103000,104000,105000,106000,107000,108000,109000,110000,111000,112000,113000,114000,115000,116000,117000,118000,119000,120000,121000,122000,123000,124000,125000,126000,127000,128000,129000,130000,131000,132000,133000,134000,135000,136000,137000,138000,139000,140000,141000,142000,143000,144000,145000,146000,147000,148000,149000,150000,151000,152000,153000,154000,155000,156000,157000,158000,159000,160000,161000,162000,163000,164000,165000,166000,167000,168000,169000,170000,171000,172000,173000,174000,175000,176000,177000,178000,179000,180000,181000,182000,183000,184000,185000,186000,187000,188000,189000,190000,191000,192000,193000,194000,195000,196000,197000,198000,199000,200000,201000,202000,203000,204000,205000,206000,207000,208000,209000,210000,211000,212000,213000,214000,215000,216000,217000,218000,219000,220000,221000,222000,223000,224000,225000,226000,227000,228000,229000,230000,231000,232000,233000,234000,235000,236000,237000,238000,239000,240000,241000,242000,243000,244000,245000,246000,247000,248000,249000,250000,251000,252000,253000,254000,255000,256000,257000,258000,259000,260000,261000,262000,263000,264000,265000,266000,267000,268000,269000,270000,271000,272000,273000,274000,275000,276000,277000,278000,279000,280000,281000,282000,283000,284000,285000,286000,287000,288000,289000,290000,291000,292000,293000,294000,295000,296000,297000,298000,299000,300000,301000,302000,303000,304000,305000,306000,307000,308000,309000,310000,311000,312000,313000,314000,315000,316000,317000,318000,319000,320000,321000,322000,323000,324000,325000,326000,327000,328000,329000,330000,331000,332000,333000,334000,335000,336000,337000,338000,339000,340000,341000,342000,343000,344000,345000,346000,347000,348000,349000,350000,351000,352000,353000,354000,355000,356000,357000,358000,359000,360000,361000,362000,363000,364000,365000,366000,367000,368000,369000,370000,371000,372000,373000,374000,375000,376000,377000,378000,379000,380000,381000,382000,383000,384000,385000,386000,387000,388000,389000,390000,391000,392000,393000,394000,395000,396000,397000,398000,399000,400000],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"index\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"value\"},\"type\":\"log\"},\"legend\":{\"title\":{\"text\":\"variable\"},\"tracegroupgap\":0},\"title\":{\"text\":\"Checkpoint Values for stanford-gpt2-small-a (Log scale)\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('9b48617d-0936-4432-a9e6-abbcbfa6eda0');\n",
+       "var gd = document.getElementById('922fb711-b683-41df-bedc-350328d71e57');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -3234,9 +3227,9 @@
        "<head><meta charset=\"utf-8\" /></head>\n",
        "<body>\n",
        "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
-       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"e382771d-4fc5-4455-9679-fc20a7ee3b0c\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"e382771d-4fc5-4455-9679-fc20a7ee3b0c\")) {                    Plotly.newPlot(                        \"e382771d-4fc5-4455-9679-fc20a7ee3b0c\",                        [{\"hovertemplate\":\"variable=0\\u003cbr\\u003eindex=%{x}\\u003cbr\\u003evalue=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"0\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines+markers\",\"name\":\"0\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49],\"xaxis\":\"x\",\"y\":[832,1664,2496,3328,4160,4992,5824,6656,7488,8320,9152,9984,10816,11648,12480,13312,14144,14976,15808,16640,17472,18304,19136,19968,20800,21632,22464,23296,24128,24960,25792,26624,27456,28288,29120,29952,30784,31616,32448,33280,34112,34944,35776,36608,37440,38272,39104,39936,40768,41600],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"index\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"value\"}},\"legend\":{\"title\":{\"text\":\"variable\"},\"tracegroupgap\":0},\"title\":{\"text\":\"Checkpoint Values for solu-1l-pile (Linear scale)\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"c9c2aa36-ee52-483a-bc6c-917a627d3f7e\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"c9c2aa36-ee52-483a-bc6c-917a627d3f7e\")) {                    Plotly.newPlot(                        \"c9c2aa36-ee52-483a-bc6c-917a627d3f7e\",                        [{\"hovertemplate\":\"variable=0\\u003cbr\\u003eindex=%{x}\\u003cbr\\u003evalue=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"0\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines+markers\",\"name\":\"0\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49],\"xaxis\":\"x\",\"y\":[832,1664,2496,3328,4160,4992,5824,6656,7488,8320,9152,9984,10816,11648,12480,13312,14144,14976,15808,16640,17472,18304,19136,19968,20800,21632,22464,23296,24128,24960,25792,26624,27456,28288,29120,29952,30784,31616,32448,33280,34112,34944,35776,36608,37440,38272,39104,39936,40768,41600],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"index\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"value\"}},\"legend\":{\"title\":{\"text\":\"variable\"},\"tracegroupgap\":0},\"title\":{\"text\":\"Checkpoint Values for solu-1l-pile (Linear scale)\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('e382771d-4fc5-4455-9679-fc20a7ee3b0c');\n",
+       "var gd = document.getElementById('c9c2aa36-ee52-483a-bc6c-917a627d3f7e');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -3273,9 +3266,9 @@
        "<head><meta charset=\"utf-8\" /></head>\n",
        "<body>\n",
        "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
-       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"c97a30e3-81bb-4a39-8684-28490def05f7\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"c97a30e3-81bb-4a39-8684-28490def05f7\")) {                    Plotly.newPlot(                        \"c97a30e3-81bb-4a39-8684-28490def05f7\",                        [{\"hovertemplate\":\"variable=0\\u003cbr\\u003eindex=%{x}\\u003cbr\\u003evalue=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"0\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines+markers\",\"name\":\"0\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99],\"xaxis\":\"x\",\"y\":[326,652,978,1304,1630,1956,2282,2608,2934,3260,3586,3912,4238,4564,4890,5216,5542,5868,6194,6520,6846,7172,7498,7824,8150,8476,8802,9128,9454,9780,10106,10432,10758,11084,11410,11736,12062,12388,12714,13040,13366,13692,14018,14344,14670,14996,15322,15648,15974,16300,16626,16952,17278,17604,17930,18256,18582,18908,19234,19560,19886,20212,20538,20864,21190,21516,21842,22168,22494,22820,23146,23472,23798,24124,24450,24776,25102,25428,25754,26080,26406,26732,27058,27384,27710,28036,28362,28688,29014,29340,29666,29992,30318,30644,30970,31296,31622,31948,32274,32600],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"index\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"value\"}},\"legend\":{\"title\":{\"text\":\"variable\"},\"tracegroupgap\":0},\"title\":{\"text\":\"Checkpoint Values for solu-6l-pile (Linear scale)\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"866c87d0-3047-4894-870f-525c60a8f2ad\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"866c87d0-3047-4894-870f-525c60a8f2ad\")) {                    Plotly.newPlot(                        \"866c87d0-3047-4894-870f-525c60a8f2ad\",                        [{\"hovertemplate\":\"variable=0\\u003cbr\\u003eindex=%{x}\\u003cbr\\u003evalue=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"0\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines+markers\",\"name\":\"0\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99],\"xaxis\":\"x\",\"y\":[326,652,978,1304,1630,1956,2282,2608,2934,3260,3586,3912,4238,4564,4890,5216,5542,5868,6194,6520,6846,7172,7498,7824,8150,8476,8802,9128,9454,9780,10106,10432,10758,11084,11410,11736,12062,12388,12714,13040,13366,13692,14018,14344,14670,14996,15322,15648,15974,16300,16626,16952,17278,17604,17930,18256,18582,18908,19234,19560,19886,20212,20538,20864,21190,21516,21842,22168,22494,22820,23146,23472,23798,24124,24450,24776,25102,25428,25754,26080,26406,26732,27058,27384,27710,28036,28362,28688,29014,29340,29666,29992,30318,30644,30970,31296,31622,31948,32274,32600],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"index\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"value\"}},\"legend\":{\"title\":{\"text\":\"variable\"},\"tracegroupgap\":0},\"title\":{\"text\":\"Checkpoint Values for solu-6l-pile (Linear scale)\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('c97a30e3-81bb-4a39-8684-28490def05f7');\n",
+       "var gd = document.getElementById('866c87d0-3047-4894-870f-525c60a8f2ad');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -3350,10 +3343,10 @@
    "execution_count": 58,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:42:36.353433Z",
-     "iopub.status.busy": "2024-09-05T17:42:36.353106Z",
-     "iopub.status.idle": "2024-09-05T17:42:36.356428Z",
-     "shell.execute_reply": "2024-09-05T17:42:36.355852Z"
+     "iopub.execute_input": "2024-09-10T17:16:11.195925Z",
+     "iopub.status.busy": "2024-09-10T17:16:11.195502Z",
+     "iopub.status.idle": "2024-09-10T17:16:11.198595Z",
+     "shell.execute_reply": "2024-09-10T17:16:11.198154Z"
     }
    },
    "outputs": [],
@@ -3380,10 +3373,10 @@
    "execution_count": 59,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:42:36.358553Z",
-     "iopub.status.busy": "2024-09-05T17:42:36.358101Z",
-     "iopub.status.idle": "2024-09-05T17:42:36.361529Z",
-     "shell.execute_reply": "2024-09-05T17:42:36.360995Z"
+     "iopub.execute_input": "2024-09-10T17:16:11.200512Z",
+     "iopub.status.busy": "2024-09-10T17:16:11.200349Z",
+     "iopub.status.idle": "2024-09-10T17:16:11.203538Z",
+     "shell.execute_reply": "2024-09-10T17:16:11.203116Z"
     }
    },
    "outputs": [],
@@ -3415,10 +3408,10 @@
    "execution_count": 60,
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2024-09-05T17:42:36.363647Z",
-     "iopub.status.busy": "2024-09-05T17:42:36.363386Z",
-     "iopub.status.idle": "2024-09-05T17:42:36.394604Z",
-     "shell.execute_reply": "2024-09-05T17:42:36.393998Z"
+     "iopub.execute_input": "2024-09-10T17:16:11.205534Z",
+     "iopub.status.busy": "2024-09-10T17:16:11.205216Z",
+     "iopub.status.idle": "2024-09-10T17:16:11.234688Z",
+     "shell.execute_reply": "2024-09-10T17:16:11.234245Z"
     }
    },
    "outputs": [
@@ -3429,9 +3422,9 @@
        "<head><meta charset=\"utf-8\" /></head>\n",
        "<body>\n",
        "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
-       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"a9eebf5f-2f5e-447e-90de-cbb3a18ef60b\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"a9eebf5f-2f5e-447e-90de-cbb3a18ef60b\")) {                    Plotly.newPlot(                        \"a9eebf5f-2f5e-447e-90de-cbb3a18ef60b\",                        [],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"value\"},\"type\":\"log\"},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"index\"}},\"legend\":{\"tracegroupgap\":0},\"title\":{\"text\":\"Induction Loss over training: solu-2l\"}},                        {\"responsive\": true}                    ).then(function(){\n",
+       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.30.0.min.js\"></script>                <div id=\"b9f31b41-2055-4494-8578-defbee4966cc\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"b9f31b41-2055-4494-8578-defbee4966cc\")) {                    Plotly.newPlot(                        \"b9f31b41-2055-4494-8578-defbee4966cc\",                        [],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"value\"},\"type\":\"log\"},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"index\"}},\"legend\":{\"tracegroupgap\":0},\"title\":{\"text\":\"Induction Loss over training: solu-2l\"}},                        {\"responsive\": true}                    ).then(function(){\n",
        "                            \n",
-       "var gd = document.getElementById('a9eebf5f-2f5e-447e-90de-cbb3a18ef60b');\n",
+       "var gd = document.getElementById('b9f31b41-2055-4494-8578-defbee4966cc');\n",
        "var x = new MutationObserver(function (mutations, observer) {{\n",
        "        var display = window.getComputedStyle(gd).display;\n",
        "        if (!display || display === 'none') {{\n",
@@ -3493,76 +3486,31 @@
   "widgets": {
    "application/vnd.jupyter.widget-state+json": {
     "state": {
-     "01811a7afc2b407fa4c58caf11506bf7": {
-      "model_module": "@jupyter-widgets/base",
-      "model_module_version": "2.0.0",
-      "model_name": "LayoutModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/base",
-       "_model_module_version": "2.0.0",
-       "_model_name": "LayoutModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "LayoutView",
-       "align_content": null,
-       "align_items": null,
-       "align_self": null,
-       "border_bottom": null,
-       "border_left": null,
-       "border_right": null,
-       "border_top": null,
-       "bottom": null,
-       "display": null,
-       "flex": null,
-       "flex_flow": null,
-       "grid_area": null,
-       "grid_auto_columns": null,
-       "grid_auto_flow": null,
-       "grid_auto_rows": null,
-       "grid_column": null,
-       "grid_gap": null,
-       "grid_row": null,
-       "grid_template_areas": null,
-       "grid_template_columns": null,
-       "grid_template_rows": null,
-       "height": null,
-       "justify_content": null,
-       "justify_items": null,
-       "left": null,
-       "margin": null,
-       "max_height": null,
-       "max_width": null,
-       "min_height": null,
-       "min_width": null,
-       "object_fit": null,
-       "object_position": null,
-       "order": null,
-       "overflow": null,
-       "padding": null,
-       "right": null,
-       "top": null,
-       "visibility": null,
-       "width": null
-      }
-     },
-     "028e71eaf7884a1880a18fbdb6a38252": {
+     "008a6d643cb74e9da094f214bdade5b2": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "ProgressStyleModel",
+      "model_name": "HBoxModel",
       "state": {
+       "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "ProgressStyleModel",
+       "_model_name": "HBoxModel",
        "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
+       "_view_module": "@jupyter-widgets/controls",
        "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "bar_color": null,
-       "description_width": ""
+       "_view_name": "HBoxView",
+       "box_style": "",
+       "children": [
+        "IPY_MODEL_6cee7db00ffb4c01bb6dd17bf0c55dc8",
+        "IPY_MODEL_95b475a20b204d5083a48b82a6f8382d",
+        "IPY_MODEL_805a8f8fcf3e4378a39ce356b3c7c23e"
+       ],
+       "layout": "IPY_MODEL_311dee65e0594c0c9be87cac3ff30b6c",
+       "tabbable": null,
+       "tooltip": null
       }
      },
-     "0411b4e36025480fb3916c53af875780": {
+     "02b561a4fcbc4ff78b7b6b8ad0e896fb": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HTMLStyleModel",
@@ -3580,41 +3528,30 @@
        "text_color": null
       }
      },
-     "055a5114eef14f54946752c86917371d": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "ProgressStyleModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "ProgressStyleModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "bar_color": null,
-       "description_width": ""
-      }
-     },
-     "096cc4aaadb64283bd8a6660dfe35bb9": {
+     "0516cd1f75a04889997cb7327a9fdf3c": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "HTMLStyleModel",
+      "model_name": "HTMLModel",
       "state": {
+       "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "HTMLStyleModel",
+       "_model_name": "HTMLModel",
        "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
+       "_view_module": "@jupyter-widgets/controls",
        "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "background": null,
-       "description_width": "",
-       "font_size": null,
-       "text_color": null
+       "_view_name": "HTMLView",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_8aa14118988e4695b82ec46fe96aa5e9",
+       "placeholder": "​",
+       "style": "IPY_MODEL_fb6a6a3f02be468c877ae97385dc5f12",
+       "tabbable": null,
+       "tooltip": null,
+       "value": "model.safetensors: 100%"
       }
      },
-     "0c173e347b6041b0875f9d2c303b3492": {
+     "0540d008d7ad474995cbc1f97e500d77": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -3667,7 +3604,31 @@
        "width": null
       }
      },
-     "0d01a56cb3dc4a3eab1a902fb310d8a2": {
+     "0688c83589ce4772b743149fd2bfabce": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HBoxModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HBoxModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "HBoxView",
+       "box_style": "",
+       "children": [
+        "IPY_MODEL_d7c2d402ee1e4e9190ffee2109a501f1",
+        "IPY_MODEL_1588536871044d748718349a6c4ee375",
+        "IPY_MODEL_4dd95b217ba34fae83ec141d5f2d0a4b"
+       ],
+       "layout": "IPY_MODEL_1d57c4e90fa74346bc732037c622ee7a",
+       "tabbable": null,
+       "tooltip": null
+      }
+     },
+     "10567f086d8f46f5990a99a3731324fa": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -3720,7 +3681,51 @@
        "width": null
       }
      },
-     "115505dc47284f0cb6169d331509ef32": {
+     "1588536871044d748718349a6c4ee375": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "FloatProgressModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "FloatProgressModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "ProgressView",
+       "bar_style": "success",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_c2f46efa04404481889ef8a7867da37d",
+       "max": 26.0,
+       "min": 0.0,
+       "orientation": "horizontal",
+       "style": "IPY_MODEL_43b2e32b46f24994a24aeb9e5b160170",
+       "tabbable": null,
+       "tooltip": null,
+       "value": 26.0
+      }
+     },
+     "17da7e9af1c347c28c312e42a4efece7": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLStyleModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "StyleView",
+       "background": null,
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
+      }
+     },
+     "18ad3bc6dccd4d7ba1dcb8f5c67e96ea": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HTMLModel",
@@ -3735,15 +3740,31 @@
        "_view_name": "HTMLView",
        "description": "",
        "description_allow_html": false,
-       "layout": "IPY_MODEL_f59255f1494b47948b5460064e7c3abc",
+       "layout": "IPY_MODEL_4477b92240f94ad78c7423f91ac9eff5",
        "placeholder": "​",
-       "style": "IPY_MODEL_44380ddd83d840b585e07399013c9c29",
+       "style": "IPY_MODEL_d9d197ca62994c82947369a275890068",
        "tabbable": null,
        "tooltip": null,
-       "value": " 124/124 [00:00&lt;00:00, 22.2kB/s]"
+       "value": "tokenizer.json: 100%"
+      }
+     },
+     "18d9cd9c263d4202b75fbc5ead7a91da": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "ProgressStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "ProgressStyleModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "StyleView",
+       "bar_color": null,
+       "description_width": ""
       }
      },
-     "13940d7c73eb495aa70fefdeb6d7ec55": {
+     "1d57c4e90fa74346bc732037c622ee7a": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -3796,7 +3817,49 @@
        "width": null
       }
      },
-     "1573df0e781a416db610e58ef2fd2d0a": {
+     "237bab3bdf234e028ba70f945b671f91": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLStyleModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "StyleView",
+       "background": null,
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
+      }
+     },
+     "27f33facb9e34151b8dce0f220cad270": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HBoxModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HBoxModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "HBoxView",
+       "box_style": "",
+       "children": [
+        "IPY_MODEL_47b8804c26bc4c38bab0a6a682a080a0",
+        "IPY_MODEL_bd0f674f4df64ddaacd86248f49980ac",
+        "IPY_MODEL_6fcd6a740f5043168d7697b1e10de7fe"
+       ],
+       "layout": "IPY_MODEL_cb4cd3672cc54abc92f62efed4a71ded",
+       "tabbable": null,
+       "tooltip": null
+      }
+     },
+     "28b414de4050442f9e8dcf15d5cd7039": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HTMLStyleModel",
@@ -3814,7 +3877,7 @@
        "text_color": null
       }
      },
-     "15aa6083796b453b8a9f32e1db888752": {
+     "311dee65e0594c0c9be87cac3ff30b6c": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -3867,7 +3930,23 @@
        "width": null
       }
      },
-     "165200e8e98047229e09b70f281a34d8": {
+     "33010587d7c44fe3a1efdc8290ade333": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "ProgressStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "ProgressStyleModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "StyleView",
+       "bar_color": null,
+       "description_width": ""
+      }
+     },
+     "3370afffa6ec450aa774360d4594d0f0": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -3920,42 +3999,16 @@
        "width": null
       }
      },
-     "194a762a1d14455bad5c75294d79ef81": {
+     "34a93cfc48a2454082ab382a7160b571": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "FloatProgressModel",
+      "model_name": "HTMLStyleModel",
       "state": {
-       "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "FloatProgressModel",
+       "_model_name": "HTMLStyleModel",
        "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "ProgressView",
-       "bar_style": "success",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_e5b32706429e41539f524d47ea4f705a",
-       "max": 1355256.0,
-       "min": 0.0,
-       "orientation": "horizontal",
-       "style": "IPY_MODEL_9685337c8c4440f9b6d04307abbf9e0e",
-       "tabbable": null,
-       "tooltip": null,
-       "value": 1355256.0
-      }
-     },
-     "1bb496a0b62e46b7a6a38f64c6f62b3a": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLStyleModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLStyleModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
+       "_view_module": "@jupyter-widgets/base",
        "_view_module_version": "2.0.0",
        "_view_name": "StyleView",
        "background": null,
@@ -3964,56 +4017,7 @@
        "text_color": null
       }
      },
-     "232a3ab348284b429955adb9ce9b6a92": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "HTMLView",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_6c5f0b95033347c6bb99585ee8623a82",
-       "placeholder": "​",
-       "style": "IPY_MODEL_7660b779de4941388ef8e0086abfbf57",
-       "tabbable": null,
-       "tooltip": null,
-       "value": "100%"
-      }
-     },
-     "250c17fa1cd749caa3494c908881efad": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "FloatProgressModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "FloatProgressModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "ProgressView",
-       "bar_style": "success",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_165200e8e98047229e09b70f281a34d8",
-       "max": 1042301.0,
-       "min": 0.0,
-       "orientation": "horizontal",
-       "style": "IPY_MODEL_028e71eaf7884a1880a18fbdb6a38252",
-       "tabbable": null,
-       "tooltip": null,
-       "value": 1042301.0
-      }
-     },
-     "2ccf74a736494e378f51375790cdc75b": {
+     "3d5179253d12488e898aa47bf606dfe0": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -4066,7 +4070,64 @@
        "width": null
       }
      },
-     "2fab1eae450f479eb2db64cd3f12ed02": {
+     "410b5e8ea7a74e53a4710f63bec5c46a": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "HTMLView",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_48458fc57304401b90253697e647dbd8",
+       "placeholder": "​",
+       "style": "IPY_MODEL_4315cacb349a45078d9a20a06d48a8a0",
+       "tabbable": null,
+       "tooltip": null,
+       "value": " 1.36M/1.36M [00:00&lt;00:00, 38.7MB/s]"
+      }
+     },
+     "4315cacb349a45078d9a20a06d48a8a0": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLStyleModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "StyleView",
+       "background": null,
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
+      }
+     },
+     "43b2e32b46f24994a24aeb9e5b160170": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "ProgressStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "ProgressStyleModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "StyleView",
+       "bar_color": null,
+       "description_width": ""
+      }
+     },
+     "4477b92240f94ad78c7423f91ac9eff5": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -4119,7 +4180,7 @@
        "width": null
       }
      },
-     "322463a77fcc4ff4afe1f15b79f65fca": {
+     "447d8c1ba33641338c516f181b3ba8c7": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -4172,49 +4233,30 @@
        "width": null
       }
      },
-     "32687fd245d5475c8dfa47143f4669a9": {
+     "47b8804c26bc4c38bab0a6a682a080a0": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "HTMLStyleModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLStyleModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "background": null,
-       "description_width": "",
-       "font_size": null,
-       "text_color": null
-      }
-     },
-     "3829b426693a411babc613fc1d68b1c8": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HBoxModel",
+      "model_name": "HTMLModel",
       "state": {
        "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "HBoxModel",
+       "_model_name": "HTMLModel",
        "_view_count": null,
        "_view_module": "@jupyter-widgets/controls",
        "_view_module_version": "2.0.0",
-       "_view_name": "HBoxView",
-       "box_style": "",
-       "children": [
-        "IPY_MODEL_96fa2acab8fe4ebeb2e3c5faee788754",
-        "IPY_MODEL_bc171be6fc9e4023a280df273ea78c7c",
-        "IPY_MODEL_8852cffea1d647bc969df746c022c33e"
-       ],
-       "layout": "IPY_MODEL_a3debcca1d614acfab710a737555ee78",
+       "_view_name": "HTMLView",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_8ece1ded3c794772bd30777f08400130",
+       "placeholder": "​",
+       "style": "IPY_MODEL_b95de56d838e48b3b5a86f0d727fc1ec",
        "tabbable": null,
-       "tooltip": null
+       "tooltip": null,
+       "value": "vocab.json: 100%"
       }
      },
-     "3b2a4f2b968b4b788fa33b7258842629": {
+     "48458fc57304401b90253697e647dbd8": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -4267,7 +4309,30 @@
        "width": null
       }
      },
-     "3e9e97a1b1f2449792386a93d3ef9da9": {
+     "4985934faaea4564bff9cdf370f1d3dc": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "HTMLView",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_10567f086d8f46f5990a99a3731324fa",
+       "placeholder": "​",
+       "style": "IPY_MODEL_f0eb65695a344d84b0e6f62f40160520",
+       "tabbable": null,
+       "tooltip": null,
+       "value": " 12/12 [00:20&lt;00:00,  1.69s/it]"
+      }
+     },
+     "4d9487d84e154880b53742b321132162": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -4320,73 +4385,30 @@
        "width": null
       }
      },
-     "40e09b2a289644d4b6031bd159605d65": {
+     "4dd95b217ba34fae83ec141d5f2d0a4b": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "HBoxModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HBoxModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "HBoxView",
-       "box_style": "",
-       "children": [
-        "IPY_MODEL_d243f9a93c5a48f2b7a26a83d8f8d5d8",
-        "IPY_MODEL_5244d573d015449085961de4b348bd21",
-        "IPY_MODEL_b2882f0d52c848a0a48b546d96b933a6"
-       ],
-       "layout": "IPY_MODEL_0d01a56cb3dc4a3eab1a902fb310d8a2",
-       "tabbable": null,
-       "tooltip": null
-      }
-     },
-     "43da9025ada5462c9ceacafb0040d9e5": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HBoxModel",
+      "model_name": "HTMLModel",
       "state": {
        "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "HBoxModel",
+       "_model_name": "HTMLModel",
        "_view_count": null,
        "_view_module": "@jupyter-widgets/controls",
        "_view_module_version": "2.0.0",
-       "_view_name": "HBoxView",
-       "box_style": "",
-       "children": [
-        "IPY_MODEL_9326ff21288f448f89edcf15a115fee9",
-        "IPY_MODEL_5315a350c488485bb93cf6d1e571e6fb",
-        "IPY_MODEL_f957f6783dde4a2e8798c7f61d6e60d7"
-       ],
-       "layout": "IPY_MODEL_c01db078359843f9b29c1ed34665a130",
+       "_view_name": "HTMLView",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_9d4d5924716049378a2ca41940476803",
+       "placeholder": "​",
+       "style": "IPY_MODEL_02b561a4fcbc4ff78b7b6b8ad0e896fb",
        "tabbable": null,
-       "tooltip": null
-      }
-     },
-     "44380ddd83d840b585e07399013c9c29": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLStyleModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLStyleModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "background": null,
-       "description_width": "",
-       "font_size": null,
-       "text_color": null
+       "tooltip": null,
+       "value": " 26.0/26.0 [00:00&lt;00:00, 4.58kB/s]"
       }
      },
-     "477ab2bf7b384716a706efe251c0af4c": {
+     "540faf238641448f809e912b89f0bb54": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -4439,46 +4461,7 @@
        "width": null
       }
      },
-     "49a5e94825ea4a69a197ce9ec10f51d0": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "HTMLView",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_9cf4f7f88d94419fa1982c343fb3a1f1",
-       "placeholder": "​",
-       "style": "IPY_MODEL_096cc4aaadb64283bd8a6660dfe35bb9",
-       "tabbable": null,
-       "tooltip": null,
-       "value": "vocab.json: 100%"
-      }
-     },
-     "4ac62c1c592f4a07b3845c7bb8f780e0": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "ProgressStyleModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "ProgressStyleModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "bar_color": null,
-       "description_width": ""
-      }
-     },
-     "4be5583eb28a498089221a44849a7403": {
+     "5d3afc1d48044c3baea5bf77f35c2752": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -4531,56 +4514,7 @@
        "width": null
       }
      },
-     "4e1981eef4314877bea01005cb920f32": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "HTMLView",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_3b2a4f2b968b4b788fa33b7258842629",
-       "placeholder": "​",
-       "style": "IPY_MODEL_5e773a3416684f3c966170693f88694a",
-       "tabbable": null,
-       "tooltip": null,
-       "value": "generation_config.json: 100%"
-      }
-     },
-     "5244d573d015449085961de4b348bd21": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "FloatProgressModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "FloatProgressModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "ProgressView",
-       "bar_style": "success",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_cee2581a67e94c1dafa02bb2a5f265ef",
-       "max": 456318.0,
-       "min": 0.0,
-       "orientation": "horizontal",
-       "style": "IPY_MODEL_7d4541a6431b40ea8fd8247a31f9dadb",
-       "tabbable": null,
-       "tooltip": null,
-       "value": 456318.0
-      }
-     },
-     "5315a350c488485bb93cf6d1e571e6fb": {
+     "5f23d06f2b22447586f73d329fd6f1b7": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "FloatProgressModel",
@@ -4596,17 +4530,17 @@
        "bar_style": "success",
        "description": "",
        "description_allow_html": false,
-       "layout": "IPY_MODEL_01811a7afc2b407fa4c58caf11506bf7",
+       "layout": "IPY_MODEL_f12fcdf158564aa880d46a54eea242dc",
        "max": 352824413.0,
        "min": 0.0,
        "orientation": "horizontal",
-       "style": "IPY_MODEL_fd4d1fae71734f6e814c4fd7fa8819a0",
+       "style": "IPY_MODEL_972d1265bbb44473922b191ca7aa89ab",
        "tabbable": null,
        "tooltip": null,
        "value": 352824413.0
       }
      },
-     "5e773a3416684f3c966170693f88694a": {
+     "6155a3097f1640089639d59a5fd01d55": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HTMLStyleModel",
@@ -4624,25 +4558,23 @@
        "text_color": null
       }
      },
-     "694ab2d491454882a879562487c32abe": {
+     "650b710c29684ecdb05b2f11095c67eb": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "HTMLStyleModel",
+      "model_name": "ProgressStyleModel",
       "state": {
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "HTMLStyleModel",
+       "_model_name": "ProgressStyleModel",
        "_view_count": null,
        "_view_module": "@jupyter-widgets/base",
        "_view_module_version": "2.0.0",
        "_view_name": "StyleView",
-       "background": null,
-       "description_width": "",
-       "font_size": null,
-       "text_color": null
+       "bar_color": null,
+       "description_width": ""
       }
      },
-     "6c5f0b95033347c6bb99585ee8623a82": {
+     "688ceb57a6834513a95e50899f15069f": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -4695,59 +4627,7 @@
        "width": null
       }
      },
-     "6cd874c59e2f48279e4d571fdd07a1b9": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLStyleModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLStyleModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "background": null,
-       "description_width": "",
-       "font_size": null,
-       "text_color": null
-      }
-     },
-     "7660b779de4941388ef8e0086abfbf57": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLStyleModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLStyleModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "background": null,
-       "description_width": "",
-       "font_size": null,
-       "text_color": null
-      }
-     },
-     "7d4541a6431b40ea8fd8247a31f9dadb": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "ProgressStyleModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "ProgressStyleModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "bar_color": null,
-       "description_width": ""
-      }
-     },
-     "83389a919e9c4569afc116065eb7c4f0": {
+     "6cee7db00ffb4c01bb6dd17bf0c55dc8": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HTMLModel",
@@ -4762,112 +4642,15 @@
        "_view_name": "HTMLView",
        "description": "",
        "description_allow_html": false,
-       "layout": "IPY_MODEL_15aa6083796b453b8a9f32e1db888752",
+       "layout": "IPY_MODEL_bcc763b0fc9147ed9c8db734601bd31a",
        "placeholder": "​",
-       "style": "IPY_MODEL_6cd874c59e2f48279e4d571fdd07a1b9",
+       "style": "IPY_MODEL_6155a3097f1640089639d59a5fd01d55",
        "tabbable": null,
        "tooltip": null,
-       "value": " 50/50 [00:02&lt;00:00, 22.25it/s]"
-      }
-     },
-     "839b0fe8fd1d4914a256d60dc2a52b91": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "FloatProgressModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "FloatProgressModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "ProgressView",
-       "bar_style": "success",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_4be5583eb28a498089221a44849a7403",
-       "max": 50.0,
-       "min": 0.0,
-       "orientation": "horizontal",
-       "style": "IPY_MODEL_9559f304126340de8391a00ffc2af8d2",
-       "tabbable": null,
-       "tooltip": null,
-       "value": 50.0
-      }
-     },
-     "840d3ecc67a84786a2508c01e23310ca": {
-      "model_module": "@jupyter-widgets/base",
-      "model_module_version": "2.0.0",
-      "model_name": "LayoutModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/base",
-       "_model_module_version": "2.0.0",
-       "_model_name": "LayoutModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "LayoutView",
-       "align_content": null,
-       "align_items": null,
-       "align_self": null,
-       "border_bottom": null,
-       "border_left": null,
-       "border_right": null,
-       "border_top": null,
-       "bottom": null,
-       "display": null,
-       "flex": null,
-       "flex_flow": null,
-       "grid_area": null,
-       "grid_auto_columns": null,
-       "grid_auto_flow": null,
-       "grid_auto_rows": null,
-       "grid_column": null,
-       "grid_gap": null,
-       "grid_row": null,
-       "grid_template_areas": null,
-       "grid_template_columns": null,
-       "grid_template_rows": null,
-       "height": null,
-       "justify_content": null,
-       "justify_items": null,
-       "left": null,
-       "margin": null,
-       "max_height": null,
-       "max_width": null,
-       "min_height": null,
-       "min_width": null,
-       "object_fit": null,
-       "object_position": null,
-       "order": null,
-       "overflow": null,
-       "padding": null,
-       "right": null,
-       "top": null,
-       "visibility": null,
-       "width": null
-      }
-     },
-     "855aff9f5430465ab4af6a62c05168d4": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLStyleModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLStyleModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "background": null,
-       "description_width": "",
-       "font_size": null,
-       "text_color": null
+       "value": "100%"
       }
      },
-     "87b5f4d77add4c7ca78fa2e7e6e139aa": {
+     "6fcd6a740f5043168d7697b1e10de7fe": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HTMLModel",
@@ -4882,38 +4665,41 @@
        "_view_name": "HTMLView",
        "description": "",
        "description_allow_html": false,
-       "layout": "IPY_MODEL_840d3ecc67a84786a2508c01e23310ca",
+       "layout": "IPY_MODEL_7fd52d89c83a450b94e80fba1f14b8bd",
        "placeholder": "​",
-       "style": "IPY_MODEL_0411b4e36025480fb3916c53af875780",
+       "style": "IPY_MODEL_34a93cfc48a2454082ab382a7160b571",
        "tabbable": null,
        "tooltip": null,
-       "value": " 1.36M/1.36M [00:00&lt;00:00, 6.79MB/s]"
+       "value": " 1.04M/1.04M [00:00&lt;00:00, 34.6MB/s]"
       }
      },
-     "8852cffea1d647bc969df746c022c33e": {
+     "6ffb62c075b844e3a584da1ba3604cd7": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
+      "model_name": "FloatProgressModel",
       "state": {
        "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
+       "_model_name": "FloatProgressModel",
        "_view_count": null,
        "_view_module": "@jupyter-widgets/controls",
        "_view_module_version": "2.0.0",
-       "_view_name": "HTMLView",
+       "_view_name": "ProgressView",
+       "bar_style": "success",
        "description": "",
        "description_allow_html": false,
-       "layout": "IPY_MODEL_c847691a3c11405db8eb8fbc5538f3a7",
-       "placeholder": "​",
-       "style": "IPY_MODEL_da4b8443fbbd4207ab5bab911bc774e4",
+       "layout": "IPY_MODEL_ca0e337ba4e6481ba5728bae7f11f7d7",
+       "max": 1355256.0,
+       "min": 0.0,
+       "orientation": "horizontal",
+       "style": "IPY_MODEL_33010587d7c44fe3a1efdc8290ade333",
        "tabbable": null,
        "tooltip": null,
-       "value": " 12/12 [00:20&lt;00:00,  1.70s/it]"
+       "value": 1355256.0
       }
      },
-     "8bccf23ea8614f7790636e7dd69b28b0": {
+     "70c3c51f506a4de4974cf045dc40e642": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HBoxModel",
@@ -4928,16 +4714,16 @@
        "_view_name": "HBoxView",
        "box_style": "",
        "children": [
-        "IPY_MODEL_9c8e5ec575d84ea6a5c96aa4c749570e",
-        "IPY_MODEL_194a762a1d14455bad5c75294d79ef81",
-        "IPY_MODEL_87b5f4d77add4c7ca78fa2e7e6e139aa"
+        "IPY_MODEL_0516cd1f75a04889997cb7327a9fdf3c",
+        "IPY_MODEL_5f23d06f2b22447586f73d329fd6f1b7",
+        "IPY_MODEL_f4eaf5d519f749e4b7ecef7567833681"
        ],
-       "layout": "IPY_MODEL_da0f3a0ffd2945939356d855658d90a3",
+       "layout": "IPY_MODEL_540faf238641448f809e912b89f0bb54",
        "tabbable": null,
        "tooltip": null
       }
      },
-     "8c9c505dabf74e98a237f21c65b95367": {
+     "74f9c61a4e3f413b90ed8f2547071ebf": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -4990,7 +4776,7 @@
        "width": null
       }
      },
-     "8cf7b30b11d248e8b78f1080528d4668": {
+     "7fd52d89c83a450b94e80fba1f14b8bd": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -5043,7 +4829,7 @@
        "width": null
       }
      },
-     "9326ff21288f448f89edcf15a115fee9": {
+     "805a8f8fcf3e4378a39ce356b3c7c23e": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HTMLModel",
@@ -5058,47 +4844,86 @@
        "_view_name": "HTMLView",
        "description": "",
        "description_allow_html": false,
-       "layout": "IPY_MODEL_0c173e347b6041b0875f9d2c303b3492",
+       "layout": "IPY_MODEL_d95212bf72bc45f9b942f1f07c7b3497",
        "placeholder": "​",
-       "style": "IPY_MODEL_e4ceba62766246a3880ce34c46382396",
+       "style": "IPY_MODEL_17da7e9af1c347c28c312e42a4efece7",
        "tabbable": null,
        "tooltip": null,
-       "value": "model.safetensors: 100%"
+       "value": " 50/50 [00:02&lt;00:00, 21.71it/s]"
       }
      },
-     "9559f304126340de8391a00ffc2af8d2": {
+     "8153259059cb4050b766ada3a5eac8a2": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "ProgressStyleModel",
+      "model_name": "HTMLStyleModel",
       "state": {
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "ProgressStyleModel",
+       "_model_name": "HTMLStyleModel",
        "_view_count": null,
        "_view_module": "@jupyter-widgets/base",
        "_view_module_version": "2.0.0",
        "_view_name": "StyleView",
-       "bar_color": null,
-       "description_width": ""
+       "background": null,
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
       }
      },
-     "9685337c8c4440f9b6d04307abbf9e0e": {
-      "model_module": "@jupyter-widgets/controls",
+     "82559a7cad044347acaeb563d9af1f85": {
+      "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
-      "model_name": "ProgressStyleModel",
+      "model_name": "LayoutModel",
       "state": {
-       "_model_module": "@jupyter-widgets/controls",
+       "_model_module": "@jupyter-widgets/base",
        "_model_module_version": "2.0.0",
-       "_model_name": "ProgressStyleModel",
+       "_model_name": "LayoutModel",
        "_view_count": null,
        "_view_module": "@jupyter-widgets/base",
        "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "bar_color": null,
-       "description_width": ""
+       "_view_name": "LayoutView",
+       "align_content": null,
+       "align_items": null,
+       "align_self": null,
+       "border_bottom": null,
+       "border_left": null,
+       "border_right": null,
+       "border_top": null,
+       "bottom": null,
+       "display": null,
+       "flex": null,
+       "flex_flow": null,
+       "grid_area": null,
+       "grid_auto_columns": null,
+       "grid_auto_flow": null,
+       "grid_auto_rows": null,
+       "grid_column": null,
+       "grid_gap": null,
+       "grid_row": null,
+       "grid_template_areas": null,
+       "grid_template_columns": null,
+       "grid_template_rows": null,
+       "height": null,
+       "justify_content": null,
+       "justify_items": null,
+       "left": null,
+       "margin": null,
+       "max_height": null,
+       "max_width": null,
+       "min_height": null,
+       "min_width": null,
+       "object_fit": null,
+       "object_position": null,
+       "order": null,
+       "overflow": null,
+       "padding": null,
+       "right": null,
+       "top": null,
+       "visibility": null,
+       "width": null
       }
      },
-     "96fa2acab8fe4ebeb2e3c5faee788754": {
+     "874725d00c554c14a10da4f89496cd4b": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HTMLModel",
@@ -5113,33 +4938,38 @@
        "_view_name": "HTMLView",
        "description": "",
        "description_allow_html": false,
-       "layout": "IPY_MODEL_2ccf74a736494e378f51375790cdc75b",
+       "layout": "IPY_MODEL_5d3afc1d48044c3baea5bf77f35c2752",
        "placeholder": "​",
-       "style": "IPY_MODEL_ea105ad53b6b44f9be96f599c6d06ec6",
+       "style": "IPY_MODEL_ca6db46f215a46c7940fd3473e34187b",
        "tabbable": null,
        "tooltip": null,
        "value": "100%"
       }
      },
-     "9a04f47585af4b21b30d2de91d685789": {
+     "88889e572ad148ff87a8fffdb35a3afd": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "HTMLStyleModel",
+      "model_name": "HTMLModel",
       "state": {
+       "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "HTMLStyleModel",
+       "_model_name": "HTMLModel",
        "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
+       "_view_module": "@jupyter-widgets/controls",
        "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "background": null,
-       "description_width": "",
-       "font_size": null,
-       "text_color": null
+       "_view_name": "HTMLView",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_82559a7cad044347acaeb563d9af1f85",
+       "placeholder": "​",
+       "style": "IPY_MODEL_a3a319cc10874775b1c0b734866a42ea",
+       "tabbable": null,
+       "tooltip": null,
+       "value": " 456k/456k [00:00&lt;00:00, 46.3MB/s]"
       }
      },
-     "9ab9621975614f0b81b69331fd0331f1": {
+     "8aa14118988e4695b82ec46fe96aa5e9": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -5192,30 +5022,31 @@
        "width": null
       }
      },
-     "9c8e5ec575d84ea6a5c96aa4c749570e": {
+     "8de918cf34204469a9a6abae1699ad36": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
+      "model_name": "HBoxModel",
       "state": {
        "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
+       "_model_name": "HBoxModel",
        "_view_count": null,
        "_view_module": "@jupyter-widgets/controls",
        "_view_module_version": "2.0.0",
-       "_view_name": "HTMLView",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_a1ffad5838934b0c895ba8c39cd34403",
-       "placeholder": "​",
-       "style": "IPY_MODEL_9a04f47585af4b21b30d2de91d685789",
+       "_view_name": "HBoxView",
+       "box_style": "",
+       "children": [
+        "IPY_MODEL_874725d00c554c14a10da4f89496cd4b",
+        "IPY_MODEL_cddcf25b67ea4b8f8e31b2392f4f1a4a",
+        "IPY_MODEL_4985934faaea4564bff9cdf370f1d3dc"
+       ],
+       "layout": "IPY_MODEL_4d9487d84e154880b53742b321132162",
        "tabbable": null,
-       "tooltip": null,
-       "value": "tokenizer.json: 100%"
+       "tooltip": null
       }
      },
-     "9cf4f7f88d94419fa1982c343fb3a1f1": {
+     "8ece1ded3c794772bd30777f08400130": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -5268,7 +5099,7 @@
        "width": null
       }
      },
-     "a1ffad5838934b0c895ba8c39cd34403": {
+     "8f8518460bd1486f980634dc8d69e4c4": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -5321,7 +5152,33 @@
        "width": null
       }
      },
-     "a3debcca1d614acfab710a737555ee78": {
+     "95b475a20b204d5083a48b82a6f8382d": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "FloatProgressModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "FloatProgressModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "ProgressView",
+       "bar_style": "success",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_0540d008d7ad474995cbc1f97e500d77",
+       "max": 50.0,
+       "min": 0.0,
+       "orientation": "horizontal",
+       "style": "IPY_MODEL_18d9cd9c263d4202b75fbc5ead7a91da",
+       "tabbable": null,
+       "tooltip": null,
+       "value": 50.0
+      }
+     },
+     "9694b20aca654e85874e76f287a305e7": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -5374,80 +5231,41 @@
        "width": null
       }
      },
-     "ae6c5587383a4acb854b423ba322ae80": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "FloatProgressModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "FloatProgressModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "ProgressView",
-       "bar_style": "success",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_322463a77fcc4ff4afe1f15b79f65fca",
-       "max": 26.0,
-       "min": 0.0,
-       "orientation": "horizontal",
-       "style": "IPY_MODEL_e2f3b2b4710947b19dbb733d4d21d91b",
-       "tabbable": null,
-       "tooltip": null,
-       "value": 26.0
-      }
-     },
-     "b2882f0d52c848a0a48b546d96b933a6": {
+     "972d1265bbb44473922b191ca7aa89ab": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
+      "model_name": "ProgressStyleModel",
       "state": {
-       "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
+       "_model_name": "ProgressStyleModel",
        "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
+       "_view_module": "@jupyter-widgets/base",
        "_view_module_version": "2.0.0",
-       "_view_name": "HTMLView",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_477ab2bf7b384716a706efe251c0af4c",
-       "placeholder": "​",
-       "style": "IPY_MODEL_855aff9f5430465ab4af6a62c05168d4",
-       "tabbable": null,
-       "tooltip": null,
-       "value": " 456k/456k [00:00&lt;00:00, 7.52MB/s]"
+       "_view_name": "StyleView",
+       "bar_color": null,
+       "description_width": ""
       }
      },
-     "b9efbbee427945918852a8f8351f3658": {
+     "9aa34eaa1a784bcb8c5e92794c27ecc3": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "HBoxModel",
+      "model_name": "HTMLStyleModel",
       "state": {
-       "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "HBoxModel",
+       "_model_name": "HTMLStyleModel",
        "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
+       "_view_module": "@jupyter-widgets/base",
        "_view_module_version": "2.0.0",
-       "_view_name": "HBoxView",
-       "box_style": "",
-       "children": [
-        "IPY_MODEL_4e1981eef4314877bea01005cb920f32",
-        "IPY_MODEL_e46ead82ee504133960b3c99b8f7e0d7",
-        "IPY_MODEL_115505dc47284f0cb6169d331509ef32"
-       ],
-       "layout": "IPY_MODEL_8c9c505dabf74e98a237f21c65b95367",
-       "tabbable": null,
-       "tooltip": null
+       "_view_name": "StyleView",
+       "background": null,
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
       }
      },
-     "ba1b85b866e34cc1bd55e06fe523f82c": {
+     "9d4d5924716049378a2ca41940476803": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -5500,33 +5318,46 @@
        "width": null
       }
      },
-     "bc171be6fc9e4023a280df273ea78c7c": {
+     "9f38f17b1a8243658f662b7c19cfc291": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "FloatProgressModel",
+      "model_name": "ProgressStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "ProgressStyleModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "StyleView",
+       "bar_color": null,
+       "description_width": ""
+      }
+     },
+     "a07aaaca4898479bb0086daf235ab228": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLModel",
       "state": {
        "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "FloatProgressModel",
+       "_model_name": "HTMLModel",
        "_view_count": null,
        "_view_module": "@jupyter-widgets/controls",
        "_view_module_version": "2.0.0",
-       "_view_name": "ProgressView",
-       "bar_style": "success",
+       "_view_name": "HTMLView",
        "description": "",
        "description_allow_html": false,
-       "layout": "IPY_MODEL_8cf7b30b11d248e8b78f1080528d4668",
-       "max": 12.0,
-       "min": 0.0,
-       "orientation": "horizontal",
-       "style": "IPY_MODEL_4ac62c1c592f4a07b3845c7bb8f780e0",
+       "layout": "IPY_MODEL_688ceb57a6834513a95e50899f15069f",
+       "placeholder": "​",
+       "style": "IPY_MODEL_28b414de4050442f9e8dcf15d5cd7039",
        "tabbable": null,
        "tooltip": null,
-       "value": 12.0
+       "value": " 124/124 [00:00&lt;00:00, 21.9kB/s]"
       }
      },
-     "bc777df5093d45d4a5dc88af7705a3f7": {
+     "a127007784d246c38bd3befe316fbaab": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -5579,7 +5410,85 @@
        "width": null
       }
      },
-     "c01db078359843f9b29c1ed34665a130": {
+     "a3a319cc10874775b1c0b734866a42ea": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLStyleModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "StyleView",
+       "background": null,
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
+      }
+     },
+     "a5f89503b135454e9897be952ba25bf4": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "FloatProgressModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "FloatProgressModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "ProgressView",
+       "bar_style": "success",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_d9f88fcbeac145f9a24e0d101d91caf1",
+       "max": 456318.0,
+       "min": 0.0,
+       "orientation": "horizontal",
+       "style": "IPY_MODEL_caa9978c2d3a4f2895f28c23db81350a",
+       "tabbable": null,
+       "tooltip": null,
+       "value": 456318.0
+      }
+     },
+     "b0731804db16433284acb42a5bf074a2": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLStyleModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "StyleView",
+       "background": null,
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
+      }
+     },
+     "b8094fe24f244d488a00415198028cbb": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "ProgressStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "ProgressStyleModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "StyleView",
+       "bar_color": null,
+       "description_width": ""
+      }
+     },
+     "b907ab58b5e24cba8615605ca344dba1": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -5632,7 +5541,25 @@
        "width": null
       }
      },
-     "c2e2458e31524312bdc7ea8c65228182": {
+     "b95de56d838e48b3b5a86f0d727fc1ec": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLStyleModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "StyleView",
+       "background": null,
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
+      }
+     },
+     "b9d76ce76d0f48c1bab169245339cc52": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -5675,35 +5602,17 @@
        "min_height": null,
        "min_width": null,
        "object_fit": null,
-       "object_position": null,
-       "order": null,
-       "overflow": null,
-       "padding": null,
-       "right": null,
-       "top": null,
-       "visibility": null,
-       "width": null
-      }
-     },
-     "c68354899e65466ca137578d798e2cd0": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLStyleModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLStyleModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "background": null,
-       "description_width": "",
-       "font_size": null,
-       "text_color": null
+       "object_position": null,
+       "order": null,
+       "overflow": null,
+       "padding": null,
+       "right": null,
+       "top": null,
+       "visibility": null,
+       "width": null
       }
      },
-     "c847691a3c11405db8eb8fbc5538f3a7": {
+     "bcc763b0fc9147ed9c8db734601bd31a": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -5756,7 +5665,33 @@
        "width": null
       }
      },
-     "cad219cb55d34a84a8bd505e58157885": {
+     "bd0f674f4df64ddaacd86248f49980ac": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "FloatProgressModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "FloatProgressModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "ProgressView",
+       "bar_style": "success",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_b9d76ce76d0f48c1bab169245339cc52",
+       "max": 1042301.0,
+       "min": 0.0,
+       "orientation": "horizontal",
+       "style": "IPY_MODEL_9f38f17b1a8243658f662b7c19cfc291",
+       "tabbable": null,
+       "tooltip": null,
+       "value": 1042301.0
+      }
+     },
+     "c2024b930e0f46828f9f1935807c38fd": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HBoxModel",
@@ -5771,16 +5706,16 @@
        "_view_name": "HBoxView",
        "box_style": "",
        "children": [
-        "IPY_MODEL_e557528f972e45e1b5ece9fd2b6f0ee6",
-        "IPY_MODEL_ae6c5587383a4acb854b423ba322ae80",
-        "IPY_MODEL_dfabbde516be47c5b62569906c3e982f"
+        "IPY_MODEL_dfd44a1536764bb291c34f3d7ee5e8f8",
+        "IPY_MODEL_c7f63f2ff6ce48cea1c9ef6a477116a4",
+        "IPY_MODEL_a07aaaca4898479bb0086daf235ab228"
        ],
-       "layout": "IPY_MODEL_e220b630f756444aa598cd8265a6fc04",
+       "layout": "IPY_MODEL_3d5179253d12488e898aa47bf606dfe0",
        "tabbable": null,
        "tooltip": null
       }
      },
-     "cee2581a67e94c1dafa02bb2a5f265ef": {
+     "c2f46efa04404481889ef8a7867da37d": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -5833,53 +5768,33 @@
        "width": null
       }
      },
-     "d243f9a93c5a48f2b7a26a83d8f8d5d8": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "HTMLView",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_c2e2458e31524312bdc7ea8c65228182",
-       "placeholder": "​",
-       "style": "IPY_MODEL_c68354899e65466ca137578d798e2cd0",
-       "tabbable": null,
-       "tooltip": null,
-       "value": "merges.txt: 100%"
-      }
-     },
-     "d83569bf74c141cfbdf849a272ad616b": {
+     "c7f63f2ff6ce48cea1c9ef6a477116a4": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
+      "model_name": "FloatProgressModel",
       "state": {
        "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
+       "_model_name": "FloatProgressModel",
        "_view_count": null,
        "_view_module": "@jupyter-widgets/controls",
        "_view_module_version": "2.0.0",
-       "_view_name": "HTMLView",
+       "_view_name": "ProgressView",
+       "bar_style": "success",
        "description": "",
        "description_allow_html": false,
-       "layout": "IPY_MODEL_13940d7c73eb495aa70fefdeb6d7ec55",
-       "placeholder": "​",
-       "style": "IPY_MODEL_1bb496a0b62e46b7a6a38f64c6f62b3a",
+       "layout": "IPY_MODEL_447d8c1ba33641338c516f181b3ba8c7",
+       "max": 124.0,
+       "min": 0.0,
+       "orientation": "horizontal",
+       "style": "IPY_MODEL_650b710c29684ecdb05b2f11095c67eb",
        "tabbable": null,
        "tooltip": null,
-       "value": " 1.04M/1.04M [00:00&lt;00:00, 3.89MB/s]"
+       "value": 124.0
       }
      },
-     "da0f3a0ffd2945939356d855658d90a3": {
+     "ca0e337ba4e6481ba5728bae7f11f7d7": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -5932,7 +5847,7 @@
        "width": null
       }
      },
-     "da4b8443fbbd4207ab5bab911bc774e4": {
+     "ca6db46f215a46c7940fd3473e34187b": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HTMLStyleModel",
@@ -5950,30 +5865,23 @@
        "text_color": null
       }
      },
-     "dfabbde516be47c5b62569906c3e982f": {
+     "caa9978c2d3a4f2895f28c23db81350a": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "HTMLModel",
+      "model_name": "ProgressStyleModel",
       "state": {
-       "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "HTMLModel",
+       "_model_name": "ProgressStyleModel",
        "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
+       "_view_module": "@jupyter-widgets/base",
        "_view_module_version": "2.0.0",
-       "_view_name": "HTMLView",
-       "description": "",
-       "description_allow_html": false,
-       "layout": "IPY_MODEL_ef20ddb2573249aa95c09444c24ca53c",
-       "placeholder": "​",
-       "style": "IPY_MODEL_694ab2d491454882a879562487c32abe",
-       "tabbable": null,
-       "tooltip": null,
-       "value": " 26.0/26.0 [00:00&lt;00:00, 4.76kB/s]"
+       "_view_name": "StyleView",
+       "bar_color": null,
+       "description_width": ""
       }
      },
-     "e220b630f756444aa598cd8265a6fc04": {
+     "cb4cd3672cc54abc92f62efed4a71ded": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -6026,23 +5934,60 @@
        "width": null
       }
      },
-     "e2f3b2b4710947b19dbb733d4d21d91b": {
-      "model_module": "@jupyter-widgets/controls",
+     "cbf2fd0392da454c9f0e289065e20bd5": {
+      "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
-      "model_name": "ProgressStyleModel",
+      "model_name": "LayoutModel",
       "state": {
-       "_model_module": "@jupyter-widgets/controls",
+       "_model_module": "@jupyter-widgets/base",
        "_model_module_version": "2.0.0",
-       "_model_name": "ProgressStyleModel",
+       "_model_name": "LayoutModel",
        "_view_count": null,
        "_view_module": "@jupyter-widgets/base",
        "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "bar_color": null,
-       "description_width": ""
+       "_view_name": "LayoutView",
+       "align_content": null,
+       "align_items": null,
+       "align_self": null,
+       "border_bottom": null,
+       "border_left": null,
+       "border_right": null,
+       "border_top": null,
+       "bottom": null,
+       "display": null,
+       "flex": null,
+       "flex_flow": null,
+       "grid_area": null,
+       "grid_auto_columns": null,
+       "grid_auto_flow": null,
+       "grid_auto_rows": null,
+       "grid_column": null,
+       "grid_gap": null,
+       "grid_row": null,
+       "grid_template_areas": null,
+       "grid_template_columns": null,
+       "grid_template_rows": null,
+       "height": null,
+       "justify_content": null,
+       "justify_items": null,
+       "left": null,
+       "margin": null,
+       "max_height": null,
+       "max_width": null,
+       "min_height": null,
+       "min_width": null,
+       "object_fit": null,
+       "object_position": null,
+       "order": null,
+       "overflow": null,
+       "padding": null,
+       "right": null,
+       "top": null,
+       "visibility": null,
+       "width": null
       }
      },
-     "e46ead82ee504133960b3c99b8f7e0d7": {
+     "cddcf25b67ea4b8f8e31b2392f4f1a4a": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "FloatProgressModel",
@@ -6058,35 +6003,17 @@
        "bar_style": "success",
        "description": "",
        "description_allow_html": false,
-       "layout": "IPY_MODEL_9ab9621975614f0b81b69331fd0331f1",
-       "max": 124.0,
+       "layout": "IPY_MODEL_9694b20aca654e85874e76f287a305e7",
+       "max": 12.0,
        "min": 0.0,
        "orientation": "horizontal",
-       "style": "IPY_MODEL_055a5114eef14f54946752c86917371d",
+       "style": "IPY_MODEL_b8094fe24f244d488a00415198028cbb",
        "tabbable": null,
        "tooltip": null,
-       "value": 124.0
-      }
-     },
-     "e4ceba62766246a3880ce34c46382396": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HTMLStyleModel",
-      "state": {
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HTMLStyleModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/base",
-       "_view_module_version": "2.0.0",
-       "_view_name": "StyleView",
-       "background": null,
-       "description_width": "",
-       "font_size": null,
-       "text_color": null
+       "value": 12.0
       }
      },
-     "e557528f972e45e1b5ece9fd2b6f0ee6": {
+     "d7c2d402ee1e4e9190ffee2109a501f1": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HTMLModel",
@@ -6101,15 +6028,15 @@
        "_view_name": "HTMLView",
        "description": "",
        "description_allow_html": false,
-       "layout": "IPY_MODEL_ba1b85b866e34cc1bd55e06fe523f82c",
+       "layout": "IPY_MODEL_8f8518460bd1486f980634dc8d69e4c4",
        "placeholder": "​",
-       "style": "IPY_MODEL_1573df0e781a416db610e58ef2fd2d0a",
+       "style": "IPY_MODEL_9aa34eaa1a784bcb8c5e92794c27ecc3",
        "tabbable": null,
        "tooltip": null,
        "value": "tokenizer_config.json: 100%"
       }
      },
-     "e5b32706429e41539f524d47ea4f705a": {
+     "d95212bf72bc45f9b942f1f07c7b3497": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -6162,31 +6089,7 @@
        "width": null
       }
      },
-     "e60d70eb78324541b8f9e1d0766491c8": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "2.0.0",
-      "model_name": "HBoxModel",
-      "state": {
-       "_dom_classes": [],
-       "_model_module": "@jupyter-widgets/controls",
-       "_model_module_version": "2.0.0",
-       "_model_name": "HBoxModel",
-       "_view_count": null,
-       "_view_module": "@jupyter-widgets/controls",
-       "_view_module_version": "2.0.0",
-       "_view_name": "HBoxView",
-       "box_style": "",
-       "children": [
-        "IPY_MODEL_49a5e94825ea4a69a197ce9ec10f51d0",
-        "IPY_MODEL_250c17fa1cd749caa3494c908881efad",
-        "IPY_MODEL_d83569bf74c141cfbdf849a272ad616b"
-       ],
-       "layout": "IPY_MODEL_3e9e97a1b1f2449792386a93d3ef9da9",
-       "tabbable": null,
-       "tooltip": null
-      }
-     },
-     "ea105ad53b6b44f9be96f599c6d06ec6": {
+     "d9d197ca62994c82947369a275890068": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HTMLStyleModel",
@@ -6204,7 +6107,7 @@
        "text_color": null
       }
      },
-     "ef20ddb2573249aa95c09444c24ca53c": {
+     "d9f88fcbeac145f9a24e0d101d91caf1": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -6257,7 +6160,30 @@
        "width": null
       }
      },
-     "f1b89201b64941828ea8f7bb5ebac2ca": {
+     "dfd44a1536764bb291c34f3d7ee5e8f8": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "HTMLView",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_cbf2fd0392da454c9f0e289065e20bd5",
+       "placeholder": "​",
+       "style": "IPY_MODEL_b0731804db16433284acb42a5bf074a2",
+       "tabbable": null,
+       "tooltip": null,
+       "value": "generation_config.json: 100%"
+      }
+     },
+     "e4a807366560448bab5fc25c6deea451": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HBoxModel",
@@ -6272,16 +6198,57 @@
        "_view_name": "HBoxView",
        "box_style": "",
        "children": [
-        "IPY_MODEL_232a3ab348284b429955adb9ce9b6a92",
-        "IPY_MODEL_839b0fe8fd1d4914a256d60dc2a52b91",
-        "IPY_MODEL_83389a919e9c4569afc116065eb7c4f0"
+        "IPY_MODEL_18ad3bc6dccd4d7ba1dcb8f5c67e96ea",
+        "IPY_MODEL_6ffb62c075b844e3a584da1ba3604cd7",
+        "IPY_MODEL_410b5e8ea7a74e53a4710f63bec5c46a"
        ],
-       "layout": "IPY_MODEL_2fab1eae450f479eb2db64cd3f12ed02",
+       "layout": "IPY_MODEL_3370afffa6ec450aa774360d4594d0f0",
        "tabbable": null,
        "tooltip": null
       }
      },
-     "f59255f1494b47948b5460064e7c3abc": {
+     "e77e435d623b4305bc459e7920424fd4": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLModel",
+      "state": {
+       "_dom_classes": [],
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "HTMLView",
+       "description": "",
+       "description_allow_html": false,
+       "layout": "IPY_MODEL_a127007784d246c38bd3befe316fbaab",
+       "placeholder": "​",
+       "style": "IPY_MODEL_237bab3bdf234e028ba70f945b671f91",
+       "tabbable": null,
+       "tooltip": null,
+       "value": "merges.txt: 100%"
+      }
+     },
+     "f0eb65695a344d84b0e6f62f40160520": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLStyleModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/base",
+       "_view_module_version": "2.0.0",
+       "_view_name": "StyleView",
+       "background": null,
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
+      }
+     },
+     "f12fcdf158564aa880d46a54eea242dc": {
       "model_module": "@jupyter-widgets/base",
       "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
@@ -6334,7 +6301,7 @@
        "width": null
       }
      },
-     "f957f6783dde4a2e8798c7f61d6e60d7": {
+     "f4eaf5d519f749e4b7ecef7567833681": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
       "model_name": "HTMLModel",
@@ -6349,28 +6316,54 @@
        "_view_name": "HTMLView",
        "description": "",
        "description_allow_html": false,
-       "layout": "IPY_MODEL_bc777df5093d45d4a5dc88af7705a3f7",
+       "layout": "IPY_MODEL_74f9c61a4e3f413b90ed8f2547071ebf",
        "placeholder": "​",
-       "style": "IPY_MODEL_32687fd245d5475c8dfa47143f4669a9",
+       "style": "IPY_MODEL_8153259059cb4050b766ada3a5eac8a2",
        "tabbable": null,
        "tooltip": null,
-       "value": " 353M/353M [00:02&lt;00:00, 173MB/s]"
+       "value": " 353M/353M [00:01&lt;00:00, 298MB/s]"
       }
      },
-     "fd4d1fae71734f6e814c4fd7fa8819a0": {
+     "f9a0ca4b543f47e0987f20fc87d7135a": {
       "model_module": "@jupyter-widgets/controls",
       "model_module_version": "2.0.0",
-      "model_name": "ProgressStyleModel",
+      "model_name": "HBoxModel",
       "state": {
+       "_dom_classes": [],
        "_model_module": "@jupyter-widgets/controls",
        "_model_module_version": "2.0.0",
-       "_model_name": "ProgressStyleModel",
+       "_model_name": "HBoxModel",
+       "_view_count": null,
+       "_view_module": "@jupyter-widgets/controls",
+       "_view_module_version": "2.0.0",
+       "_view_name": "HBoxView",
+       "box_style": "",
+       "children": [
+        "IPY_MODEL_e77e435d623b4305bc459e7920424fd4",
+        "IPY_MODEL_a5f89503b135454e9897be952ba25bf4",
+        "IPY_MODEL_88889e572ad148ff87a8fffdb35a3afd"
+       ],
+       "layout": "IPY_MODEL_b907ab58b5e24cba8615605ca344dba1",
+       "tabbable": null,
+       "tooltip": null
+      }
+     },
+     "fb6a6a3f02be468c877ae97385dc5f12": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLStyleModel",
+      "state": {
+       "_model_module": "@jupyter-widgets/controls",
+       "_model_module_version": "2.0.0",
+       "_model_name": "HTMLStyleModel",
        "_view_count": null,
        "_view_module": "@jupyter-widgets/base",
        "_view_module_version": "2.0.0",
        "_view_name": "StyleView",
-       "bar_color": null,
-       "description_width": ""
+       "background": null,
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
       }
      }
     },
diff --git a/searchindex.js b/searchindex.js
index 34a0a65ef..83e77d80e 100644
--- a/searchindex.js
+++ b/searchindex.js
@@ -1 +1 @@
-Search.setIndex({"docnames": ["content/citation", "content/contributing", "content/gallery", "content/getting_started", "content/getting_started_mech_interp", "content/news/release-2.0", "content/special_cases", "content/tutorials", "generated/code/modules", "generated/code/transformer_lens", "generated/code/transformer_lens.ActivationCache", "generated/code/transformer_lens.FactoredMatrix", "generated/code/transformer_lens.HookedEncoder", "generated/code/transformer_lens.HookedEncoderDecoder", "generated/code/transformer_lens.HookedTransformer", "generated/code/transformer_lens.HookedTransformerConfig", "generated/code/transformer_lens.SVDInterpreter", "generated/code/transformer_lens.components", "generated/code/transformer_lens.components.abstract_attention", "generated/code/transformer_lens.components.attention", "generated/code/transformer_lens.components.bert_block", "generated/code/transformer_lens.components.bert_embed", "generated/code/transformer_lens.components.bert_mlm_head", "generated/code/transformer_lens.components.embed", "generated/code/transformer_lens.components.grouped_query_attention", "generated/code/transformer_lens.components.layer_norm", "generated/code/transformer_lens.components.layer_norm_pre", "generated/code/transformer_lens.components.pos_embed", "generated/code/transformer_lens.components.rms_norm", "generated/code/transformer_lens.components.rms_norm_pre", "generated/code/transformer_lens.components.t5_attention", "generated/code/transformer_lens.components.t5_block", "generated/code/transformer_lens.components.token_typed_embed", "generated/code/transformer_lens.components.transformer_block", "generated/code/transformer_lens.components.unembed", "generated/code/transformer_lens.evals", "generated/code/transformer_lens.head_detector", "generated/code/transformer_lens.hook_points", "generated/code/transformer_lens.loading_from_pretrained", "generated/code/transformer_lens.past_key_value_caching", "generated/code/transformer_lens.patching", "generated/code/transformer_lens.pretrained", "generated/code/transformer_lens.pretrained.weight_conversions", "generated/code/transformer_lens.pretrained.weight_conversions.bert", "generated/code/transformer_lens.pretrained.weight_conversions.bloom", "generated/code/transformer_lens.pretrained.weight_conversions.coder", "generated/code/transformer_lens.pretrained.weight_conversions.gemma", "generated/code/transformer_lens.pretrained.weight_conversions.gpt2", "generated/code/transformer_lens.pretrained.weight_conversions.gptj", "generated/code/transformer_lens.pretrained.weight_conversions.llama", "generated/code/transformer_lens.pretrained.weight_conversions.mingpt", "generated/code/transformer_lens.pretrained.weight_conversions.mistral", "generated/code/transformer_lens.pretrained.weight_conversions.mixtral", "generated/code/transformer_lens.pretrained.weight_conversions.nanogpt", "generated/code/transformer_lens.pretrained.weight_conversions.neel_solu_old", "generated/code/transformer_lens.pretrained.weight_conversions.neo", "generated/code/transformer_lens.pretrained.weight_conversions.neox", "generated/code/transformer_lens.pretrained.weight_conversions.opt", "generated/code/transformer_lens.pretrained.weight_conversions.phi", "generated/code/transformer_lens.pretrained.weight_conversions.phi3", "generated/code/transformer_lens.pretrained.weight_conversions.qwen", "generated/code/transformer_lens.pretrained.weight_conversions.qwen2", "generated/code/transformer_lens.pretrained.weight_conversions.t5", "generated/code/transformer_lens.train", "generated/code/transformer_lens.utilities", "generated/code/transformer_lens.utilities.activation_functions", "generated/code/transformer_lens.utilities.addmm", "generated/code/transformer_lens.utilities.attention", "generated/code/transformer_lens.utilities.devices", "generated/code/transformer_lens.utils", "generated/demos/Exploratory_Analysis_Demo", "generated/demos/Main_Demo", "generated/model_properties_table", "index"], "filenames": ["content/citation.md", "content/contributing.md", "content/gallery.md", "content/getting_started.md", "content/getting_started_mech_interp.md", "content/news/release-2.0.md", "content/special_cases.md", "content/tutorials.md", "generated/code/modules.rst", "generated/code/transformer_lens.rst", "generated/code/transformer_lens.ActivationCache.rst", "generated/code/transformer_lens.FactoredMatrix.rst", "generated/code/transformer_lens.HookedEncoder.rst", "generated/code/transformer_lens.HookedEncoderDecoder.rst", "generated/code/transformer_lens.HookedTransformer.rst", "generated/code/transformer_lens.HookedTransformerConfig.rst", "generated/code/transformer_lens.SVDInterpreter.rst", "generated/code/transformer_lens.components.rst", "generated/code/transformer_lens.components.abstract_attention.rst", "generated/code/transformer_lens.components.attention.rst", "generated/code/transformer_lens.components.bert_block.rst", "generated/code/transformer_lens.components.bert_embed.rst", "generated/code/transformer_lens.components.bert_mlm_head.rst", "generated/code/transformer_lens.components.embed.rst", "generated/code/transformer_lens.components.grouped_query_attention.rst", "generated/code/transformer_lens.components.layer_norm.rst", "generated/code/transformer_lens.components.layer_norm_pre.rst", "generated/code/transformer_lens.components.pos_embed.rst", "generated/code/transformer_lens.components.rms_norm.rst", "generated/code/transformer_lens.components.rms_norm_pre.rst", "generated/code/transformer_lens.components.t5_attention.rst", "generated/code/transformer_lens.components.t5_block.rst", "generated/code/transformer_lens.components.token_typed_embed.rst", "generated/code/transformer_lens.components.transformer_block.rst", "generated/code/transformer_lens.components.unembed.rst", "generated/code/transformer_lens.evals.rst", "generated/code/transformer_lens.head_detector.rst", "generated/code/transformer_lens.hook_points.rst", "generated/code/transformer_lens.loading_from_pretrained.rst", "generated/code/transformer_lens.past_key_value_caching.rst", "generated/code/transformer_lens.patching.rst", "generated/code/transformer_lens.pretrained.rst", "generated/code/transformer_lens.pretrained.weight_conversions.rst", "generated/code/transformer_lens.pretrained.weight_conversions.bert.rst", "generated/code/transformer_lens.pretrained.weight_conversions.bloom.rst", "generated/code/transformer_lens.pretrained.weight_conversions.coder.rst", "generated/code/transformer_lens.pretrained.weight_conversions.gemma.rst", "generated/code/transformer_lens.pretrained.weight_conversions.gpt2.rst", "generated/code/transformer_lens.pretrained.weight_conversions.gptj.rst", "generated/code/transformer_lens.pretrained.weight_conversions.llama.rst", "generated/code/transformer_lens.pretrained.weight_conversions.mingpt.rst", "generated/code/transformer_lens.pretrained.weight_conversions.mistral.rst", "generated/code/transformer_lens.pretrained.weight_conversions.mixtral.rst", "generated/code/transformer_lens.pretrained.weight_conversions.nanogpt.rst", "generated/code/transformer_lens.pretrained.weight_conversions.neel_solu_old.rst", "generated/code/transformer_lens.pretrained.weight_conversions.neo.rst", "generated/code/transformer_lens.pretrained.weight_conversions.neox.rst", "generated/code/transformer_lens.pretrained.weight_conversions.opt.rst", "generated/code/transformer_lens.pretrained.weight_conversions.phi.rst", "generated/code/transformer_lens.pretrained.weight_conversions.phi3.rst", "generated/code/transformer_lens.pretrained.weight_conversions.qwen.rst", "generated/code/transformer_lens.pretrained.weight_conversions.qwen2.rst", "generated/code/transformer_lens.pretrained.weight_conversions.t5.rst", "generated/code/transformer_lens.train.rst", "generated/code/transformer_lens.utilities.rst", "generated/code/transformer_lens.utilities.activation_functions.rst", "generated/code/transformer_lens.utilities.addmm.rst", "generated/code/transformer_lens.utilities.attention.rst", "generated/code/transformer_lens.utilities.devices.rst", "generated/code/transformer_lens.utils.rst", "generated/demos/Exploratory_Analysis_Demo.ipynb", "generated/demos/Main_Demo.ipynb", "generated/model_properties_table.md", "index.md"], "titles": ["Citation", "Contributing", "Gallery", "Getting Started", "Getting Started in Mechanistic Interpretability", "TransformerLens 2.0", "Special Cases", "Tutorials", "Transformer Lens API", "transformer_lens", "transformer_lens.ActivationCache", "transformer_lens.FactoredMatrix", "transformer_lens.HookedEncoder", "transformer_lens.HookedEncoderDecoder", "transformer_lens.HookedTransformer", "transformer_lens.HookedTransformerConfig", "transformer_lens.SVDInterpreter", "transformer_lens.components", "transformer_lens.components.abstract_attention", "transformer_lens.components.attention", "transformer_lens.components.bert_block", "transformer_lens.components.bert_embed", "transformer_lens.components.bert_mlm_head", "transformer_lens.components.embed", "transformer_lens.components.grouped_query_attention", "transformer_lens.components.layer_norm", "transformer_lens.components.layer_norm_pre", "transformer_lens.components.pos_embed", "transformer_lens.components.rms_norm", "transformer_lens.components.rms_norm_pre", "transformer_lens.components.t5_attention", "transformer_lens.components.t5_block", "transformer_lens.components.token_typed_embed", "transformer_lens.components.transformer_block", "transformer_lens.components.unembed", "transformer_lens.evals", "transformer_lens.head_detector", "transformer_lens.hook_points", "transformer_lens.loading_from_pretrained", "transformer_lens.past_key_value_caching", "transformer_lens.patching", "transformer_lens.pretrained", "transformer_lens.pretrained.weight_conversions", "transformer_lens.pretrained.weight_conversions.bert", "transformer_lens.pretrained.weight_conversions.bloom", "transformer_lens.pretrained.weight_conversions.coder", "transformer_lens.pretrained.weight_conversions.gemma", "transformer_lens.pretrained.weight_conversions.gpt2", "transformer_lens.pretrained.weight_conversions.gptj", "transformer_lens.pretrained.weight_conversions.llama", "transformer_lens.pretrained.weight_conversions.mingpt", "transformer_lens.pretrained.weight_conversions.mistral", "transformer_lens.pretrained.weight_conversions.mixtral", "transformer_lens.pretrained.weight_conversions.nanogpt", "transformer_lens.pretrained.weight_conversions.neel_solu_old", "transformer_lens.pretrained.weight_conversions.neo", "transformer_lens.pretrained.weight_conversions.neox", "transformer_lens.pretrained.weight_conversions.opt", "transformer_lens.pretrained.weight_conversions.phi", "transformer_lens.pretrained.weight_conversions.phi3", "transformer_lens.pretrained.weight_conversions.qwen", "transformer_lens.pretrained.weight_conversions.qwen2", "transformer_lens.pretrained.weight_conversions.t5", "transformer_lens.train", "transformer_lens.utilities", "transformer_lens.utilities.activation_functions", "transformer_lens.utilities.addmm", "transformer_lens.utilities.attention", "transformer_lens.utilities.devices", "transformer_lens.utils", "Exploratory Analysis Demo", "Transformer Lens Main Demo Notebook", "Model Properties Table", "TransformerLens"], "terms": {"pleas": [0, 1, 3, 4, 5, 71], "cite": 0, "thi": [0, 1, 3, 4, 5, 6, 7, 10, 11, 12, 13, 14, 15, 16, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 32, 33, 34, 35, 36, 37, 38, 39, 40, 63, 67, 68, 69, 71, 73], "librari": [0, 2, 3, 4, 5, 7, 10, 35, 69, 70], "misc": 0, "nanda2022transformerlen": 0, "titl": [0, 1, 70, 71], "transformerlen": [0, 2, 3, 4, 7, 10, 14, 18, 38, 69, 70, 71], "author": [0, 70], "neel": [0, 2, 4, 7, 14, 16, 71], "nanda": [0, 2, 4, 14, 71], "joseph": [0, 5], "bloom": [0, 5, 18, 38, 42, 72], "year": 0, "2022": [0, 69], "howpublish": 0, "url": [0, 3], "http": [0, 1, 3, 7, 10, 14, 15, 18, 24, 32, 35, 36, 40, 66, 69, 70, 71], "github": [0, 1, 3, 7, 14, 66], "com": [0, 3, 7, 10, 14, 18, 66, 70, 71], "transformerlensorg": [0, 3, 7], "For": [1, 5, 10, 12, 13, 14, 18, 32, 36, 69, 70], "one": [1, 3, 4, 5, 10, 12, 13, 14, 15, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 36, 37, 38, 39, 40, 69, 70, 71, 73], "click": [1, 71], "your": [1, 3, 5, 7, 14, 15, 36, 37, 70, 71], "develop": [1, 5, 7, 70, 71], "environ": [1, 3], "project": [1, 5, 7, 10, 18, 24, 63, 70], "includ": [1, 4, 5, 7, 10, 12, 13, 14, 15, 35, 36, 37, 70], "It": [1, 3, 5, 7, 10, 12, 13, 14, 15, 18, 35, 37, 40, 69, 70, 71, 73], "can": [1, 2, 3, 5, 6, 7, 10, 11, 12, 13, 14, 15, 16, 18, 19, 24, 35, 36, 37, 38, 39, 40, 69, 70, 71, 73], "us": [1, 2, 3, 5, 6, 7, 10, 12, 13, 14, 15, 16, 18, 19, 24, 26, 31, 33, 35, 36, 37, 38, 39, 40, 63, 67, 68, 69, 70, 71, 73], "local": [1, 14, 15, 18, 19, 24, 38, 69, 71], "v": [1, 10, 12, 13, 14, 15, 18, 24, 40, 70, 71], "code": [1, 4, 5, 10, 14, 15, 18, 35, 36, 37, 38, 69, 70, 71], "codespac": 1, "poetri": 1, "packag": 1, "manag": [1, 5, 10, 14, 37, 69], "instal": [1, 5, 70, 71], "follow": [1, 3, 5, 10, 14, 69, 71, 73], "also": [1, 5, 7, 10, 12, 13, 14, 15, 16, 31, 36, 37, 38, 68, 69, 70, 71], "virtual": 1, "config": [1, 14, 15, 18, 19, 24, 38, 40, 63], "virtualenv": 1, "true": [1, 10, 12, 13, 14, 15, 30, 31, 35, 36, 37, 38, 40, 68, 69, 70, 71], "dev": 1, "doc": [1, 5, 8, 10, 14, 71], "jupyt": 1, "If": [1, 3, 5, 8, 10, 12, 13, 14, 15, 25, 28, 32, 36, 37, 38, 40, 68, 69, 70, 71], "ad": [1, 7, 14, 15, 18, 37, 70, 71], "featur": [1, 3, 5, 7, 12, 13, 16, 18, 40, 69, 70, 73], "add": [1, 5, 14, 15, 18, 37, 39, 66, 69, 70, 71, 73], "unit": [1, 5], "you": [1, 3, 4, 5, 7, 10, 12, 13, 14, 15, 16, 30, 35, 36, 37, 38, 69, 70, 71, 73], "need": [1, 3, 5, 10, 14, 15, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 69, 70, 71, 73], "model": [1, 2, 3, 6, 7, 10, 12, 13, 14, 15, 16, 18, 19, 21, 24, 35, 36, 37, 38, 40, 45, 63, 68, 69, 70], "ones": [1, 12, 13, 14, 31, 36, 70], "ar": [1, 3, 4, 5, 6, 7, 10, 11, 12, 13, 14, 15, 18, 20, 24, 35, 36, 37, 38, 39, 40, 69, 71, 73], "cach": [1, 10, 14, 15, 31, 33, 36, 37, 39, 40, 69, 70, 73], "action": [1, 2, 70], "so": [1, 5, 7, 10, 11, 12, 13, 14, 15, 16, 35, 37, 38, 39, 40, 69, 70, 71], "quickli": [1, 5, 7], "cd": [1, 70, 71], "These": [1, 70, 71], "gpt2": [1, 14, 15, 16, 18, 19, 24, 35, 38, 42, 70, 71, 72], "attn": [1, 10, 12, 13, 14, 15, 18, 30, 38, 40, 69, 70, 71, 72], "onli": [1, 2, 5, 10, 11, 12, 13, 14, 15, 18, 19, 24, 26, 31, 33, 36, 37, 38, 69, 70, 71, 72], "1l": [1, 38, 70, 71, 72], "2l": [1, 14, 38, 71, 72], "3l": [1, 38, 71, 72], "4l": [1, 38, 71, 72], "tini": [1, 10, 14, 38, 69, 70, 71, 72], "stori": [1, 10, 14, 38, 40, 69, 70, 72], "1m": [1, 10, 14, 38, 69, 72], "note": [1, 3, 5, 10, 11, 12, 13, 14, 15, 18, 24, 35, 37, 38, 66, 69, 70, 71], "i": [1, 3, 4, 5, 6, 7, 10, 11, 12, 13, 14, 15, 16, 18, 22, 24, 26, 31, 32, 33, 35, 36, 37, 38, 39, 40, 63, 67, 68, 69, 70, 73], "quit": [1, 5], "slow": [1, 71], "we": [1, 2, 5, 8, 10, 14, 15, 36, 39, 40, 69, 70, 71, 73], "have": [1, 3, 5, 10, 12, 13, 14, 18, 19, 24, 36, 40, 69, 70, 71, 73], "cpu": [1, 10, 12, 13, 14, 15, 18, 38, 70, 71], "smaller": [1, 5, 71], "like": [1, 3, 4, 5, 7, 12, 13, 14, 15, 21, 30, 35, 36, 40, 69, 70, 71, 73], "prefer": 1, "possibl": [1, 5, 12, 13, 14, 36, 40, 69, 70, 71, 73], "via": [1, 2, 3, 4, 5, 12, 13, 14, 40, 70], "make": [1, 3, 5, 7, 11, 12, 13, 14, 36, 37, 70, 71, 73], "accept": [1, 3, 5, 12, 13, 14, 37, 70], "notebook": [1, 3, 7, 70, 73], "all": [1, 4, 5, 10, 12, 13, 14, 15, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 32, 33, 34, 35, 36, 37, 40, 65, 69, 70], "suit": 1, "mention": [1, 5, 71], "pycln": 1, "isort": 1, "black": [1, 71], "pull": [1, 5], "request": [1, 5], "check": [1, 3, 5, 7, 14, 16, 18, 35, 36, 37, 69, 70, 71], "file": [1, 5, 69], "line": [1, 5, 70, 71], "length": [1, 10, 12, 13, 14, 15, 18, 25, 26, 27, 28, 29, 69, 70, 71], "set": [1, 2, 5, 10, 12, 13, 14, 15, 18, 35, 36, 37, 40, 63, 69, 70, 71], "100": [1, 35, 70, 71], "pyproject": 1, "toml": 1, "instead": [1, 6, 10, 14, 15, 18, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 70, 71], "default": [1, 6, 10, 14, 15, 16, 18, 19, 24, 27, 31, 33, 35, 36, 37, 38, 40, 69, 70, 71], "88": [1, 71], "sure": [1, 3, 5, 14, 69, 70, 71], "thorough": 1, "ani": [1, 3, 10, 12, 13, 14, 15, 18, 19, 24, 37, 69, 70, 71, 73], "should": [1, 5, 7, 10, 12, 13, 14, 18, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 36, 37, 63, 69, 70, 71], "do": [1, 3, 4, 5, 7, 10, 12, 13, 14, 18, 35, 37, 40, 69, 70, 71, 73], "directli": [1, 5, 12, 13, 15, 69, 70, 71], "automat": [1, 5, 7, 14, 15, 69, 70, 71], "gener": [1, 5, 7, 13, 14, 18, 31, 33, 35, 39, 40, 69, 70], "api": [1, 5, 37, 70], "when": [1, 3, 5, 7, 10, 11, 12, 13, 14, 15, 24, 31, 33, 35, 37, 38, 39, 40, 67, 69, 70, 71], "merg": [1, 5, 14], "main": [1, 3, 5, 6, 7, 10, 37, 70], "thei": [1, 4, 5, 14, 15, 18, 35, 40, 69, 70, 71, 73], "pytest": 1, "doctest": 1, "want": [1, 5, 7, 10, 14, 16, 35, 36, 37, 39, 69, 70, 71], "view": [1, 2], "chang": [1, 2, 3, 5, 14, 15, 37, 38, 40, 69, 70, 71], "hot": [1, 70, 71], "reload": [1, 70, 71], "give": [1, 5, 10, 14, 15, 35, 38, 40, 69, 70, 71], "real": [1, 7, 69, 70, 71, 73], "time": [1, 5, 7, 8, 10, 14, 36, 37, 69, 70, 71], "edit": [1, 7, 14, 40, 70, 71, 73], "googl": [1, 7, 38, 70, 71], "python": [1, 2, 15, 30, 35, 38, 69, 71, 72], "write": [1, 2, 3, 5, 14, 69, 70, 71, 73], "some": [1, 3, 5, 10, 14, 16, 18, 35, 37, 40, 69, 70], "from": [1, 2, 3, 4, 5, 6, 7, 10, 11, 12, 13, 14, 15, 16, 18, 24, 32, 35, 36, 38, 40, 69, 70, 71, 73], "restructuredtext": 1, "rest": [1, 5, 10, 14, 15, 35, 38, 69, 71], "In": [1, 2, 5, 6, 10, 12, 13, 14, 35, 37, 70, 71], "case": [1, 2, 5, 10, 14, 15, 35, 36, 37, 38, 40, 69, 70, 72], "A": [1, 2, 4, 5, 10, 11, 12, 13, 14, 18, 24, 31, 32, 33, 35, 37, 39, 40, 69, 70, 71], "descript": 1, "what": [1, 3, 5, 7, 10, 14, 36, 40, 71, 73], "doe": [1, 5, 10, 12, 13, 14, 15, 36, 37, 40, 69, 70, 71], "much": [1, 5, 10, 14, 35, 36, 40, 69, 70, 71], "detail": [1, 5, 10, 14, 15, 18, 24, 33, 38, 40, 69, 70, 71], "necessari": [1, 5, 71], "fulli": [1, 40, 70], "understand": [1, 10, 14, 36, 71], "warn": [1, 10, 14, 36, 37, 69], "user": [1, 2, 5, 15, 69, 71], "e": [1, 5, 10, 12, 13, 14, 15, 18, 36, 37, 38, 69, 70, 71, 72], "g": [1, 5, 10, 12, 13, 14, 36, 37, 69, 71], "common": [1, 5, 7, 10, 14, 15, 18, 69, 70, 71], "pitfal": 1, "exampl": [1, 2, 10, 12, 13, 14, 16, 18, 32, 35, 37, 69, 70], "here": [1, 2, 3, 5, 14, 15, 18, 19, 24, 35, 36, 69, 70, 71], "print": [1, 10, 35, 63, 69, 70, 71], "1": [1, 3, 4, 5, 10, 12, 13, 14, 15, 18, 19, 20, 24, 30, 31, 32, 36, 37, 38, 39, 69, 70, 71, 72], "2": [1, 3, 4, 10, 12, 13, 14, 15, 18, 35, 36, 38, 69, 70, 71, 72, 73], "3": [1, 6, 10, 11, 12, 13, 14, 18, 35, 38, 40, 69, 70, 71, 72, 73], "arg": [1, 24, 37], "param_without_type_signatur": 1, "each": [1, 5, 10, 11, 12, 13, 14, 15, 18, 36, 37, 38, 39, 40, 67, 69, 70, 71], "indent": 1, "onc": [1, 3, 5, 14, 69, 70, 71], "more": [1, 5, 7, 10, 11, 14, 15, 18, 32, 36, 40, 69, 70, 71, 73], "param_2": 1, "anoth": [1, 5, 70, 71], "paramet": [1, 5, 7, 10, 12, 13, 14, 15, 16, 18, 19, 24, 25, 26, 27, 31, 33, 35, 36, 37, 38, 40, 68, 69, 70], "return": [1, 5, 10, 11, 12, 13, 14, 18, 24, 27, 31, 33, 35, 36, 37, 38, 40, 63, 68, 69, 70, 71], "without": [1, 3, 5, 10, 14, 28, 29, 69, 70, 71], "type": [1, 6, 7, 10, 12, 13, 14, 15, 16, 21, 24, 27, 31, 32, 33, 36, 37, 38, 40, 63, 68, 69, 70, 71], "signatur": [1, 12, 13, 14, 71], "rais": [1, 14, 36, 38, 69, 71], "inform": [1, 14, 15, 32, 37, 38, 70, 71], "about": [1, 5, 7, 10, 14, 35, 37, 40, 69, 70, 71, 73], "error": [1, 10, 14, 36, 38, 71], "mai": [1, 5, 10, 12, 13, 14, 15, 18, 69, 70, 71], "part": [1, 5, 10, 14, 26, 40, 70, 71, 73], "codebas": [1, 71], "cross": [1, 10, 14, 30, 31, 69, 70, 71], "referenc": [1, 5], "omit": [1, 36, 71], "full": [1, 4, 5, 10, 12, 14, 15, 18, 69, 71], "path": [1, 4, 69], "same": [1, 3, 5, 10, 11, 14, 15, 18, 36, 37, 39, 40, 67, 69, 70, 71], "mod": 1, "transformer_len": [1, 3, 5, 8, 70, 71], "modul": [1, 5, 8, 12, 13, 14, 15, 16, 18, 19, 20, 21, 22, 23, 25, 26, 27, 28, 29, 31, 32, 33, 34, 35, 37, 38, 39, 40, 69, 71], "const": 1, "loading_from_pretrain": [1, 8, 9, 14, 71], "official_model_nam": [1, 14, 38], "hookedtransform": [1, 3, 5, 6, 7, 8, 9, 10, 12, 13, 15, 16, 35, 36, 38, 40, 63, 68, 69, 70, 71], "meth": [1, 10], "from_pretrain": [1, 3, 6, 10, 12, 13, 14, 16, 35, 38, 69, 70, 71], "attr": 1, "cfg": [1, 12, 13, 14, 16, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 39, 44, 45, 51, 52, 58, 59, 60, 61, 62, 68, 69, 70, 71], "latex": 1, "re": [1, 3, 4, 7, 10, 14, 15, 36, 70, 71], "place": [1, 5, 7, 12, 13, 14, 69, 70, 71], "string": [1, 12, 13, 14, 15, 35, 36, 37, 38, 69, 70, 71], "backward": [1, 37, 70, 71], "slash": 1, "must": [1, 10, 12, 13, 14, 15, 18, 36, 37, 66, 69, 70, 71], "repeat": [1, 14, 35, 39, 69, 70, 71], "inlin": 1, "displai": [1, 70, 71], "mode": [1, 10, 14, 15, 18, 26, 69, 70, 71], "b": [1, 11, 12, 14, 32, 35, 38, 40, 67, 69, 71, 72], "2ab": 1, "nowrap": 1, "begin": [1, 3, 5, 13, 14, 35, 36, 69, 70, 71], "eqnarrai": 1, "y": [1, 10, 14, 36, 70, 71], "ax": [1, 14, 40, 71], "bx": 1, "c": [1, 38, 69, 71, 72], "f": [1, 70, 71], "x": [1, 5, 10, 14, 18, 25, 26, 28, 29, 36, 37, 38, 66, 69, 70, 71], "2xy": 1, "end": [1, 5, 7, 14, 37, 40, 69, 70, 71], "ital": 1, "text": [1, 7, 10, 14, 15, 18, 31, 33, 35, 37, 38, 39, 69, 70], "bold": 1, "list": [1, 3, 4, 10, 12, 13, 14, 15, 35, 36, 37, 39, 40, 69, 70, 71], "item": [1, 5, 10, 69, 70, 71], "number": [1, 5, 10, 14, 15, 16, 18, 35, 38, 40, 63, 68, 69, 70, 71], "quot": 1, "level": [1, 37, 70, 71, 73], "extern": [1, 70], "link": [1, 14, 35], "domain": 1, "invalid": 1, "research": [2, 3, 4, 5, 7, 70, 71, 73], "done": [2, 4, 5, 6, 10, 14, 15, 18, 37, 70, 71], "involv": [2, 5, 70, 71], "progress": [2, 5, 14, 71], "measur": [2, 35, 36, 40, 69, 70], "grokk": [2, 7], "mechanist": [2, 3, 7, 40, 70, 71], "interpret": [2, 3, 7, 10, 14, 16, 36, 40, 69, 70], "iclr": 2, "spotlight": 2, "2023": 2, "lawrenc": 2, "chan": 2, "tom": [2, 70], "lieberum": 2, "jess": 2, "smith": 2, "jacob": 2, "steinhardt": 2, "find": [2, 5, 7, 10, 11, 14, 40, 70, 71], "neuron": [2, 7, 10, 14, 70, 71], "haystack": 2, "studi": [2, 4, 40, 70, 71], "spars": [2, 5], "probe": 2, "gurne": 2, "matthew": 2, "pauli": 2, "katherin": 2, "harvei": 2, "dmitrii": 2, "troitskii": 2, "dimitri": 2, "bertsima": 2, "toward": [2, 18, 40, 70], "autom": 2, "circuit": [2, 12, 14, 18, 35, 36, 40, 69, 70, 71], "discoveri": 2, "arthur": [2, 71], "conmi": [2, 71], "augustin": 2, "n": [2, 18, 63, 66, 69, 70, 71], "mavor": 2, "parker": 2, "aengu": 2, "lynch": 2, "stefan": 2, "heimersheim": 2, "adri\u00e0": 2, "garriga": 2, "alonso": 2, "actual": [2, 5, 14, 36, 37, 71], "othello": [2, 7, 38, 72], "gpt": [2, 3, 4, 7, 10, 12, 13, 14, 15, 18, 19, 24, 35, 38, 69, 70, 71, 72, 73], "ha": [2, 3, 4, 5, 10, 11, 12, 13, 14, 18, 24, 35, 38, 39, 40, 67, 69, 70, 71], "linear": [2, 7, 12, 13, 14, 18, 38, 67, 70, 71], "emerg": [2, 7], "world": [2, 5, 7, 71, 73], "represent": [2, 7], "docstr": 2, "4": [2, 3, 5, 15, 18, 35, 69, 70, 71, 72], "layer": [2, 6, 10, 12, 13, 14, 15, 16, 18, 19, 21, 24, 25, 26, 36, 37, 38, 39, 40, 67, 68, 69, 71], "attent": [2, 7, 10, 12, 13, 14, 15, 17, 18, 20, 24, 27, 30, 31, 33, 36, 40, 64, 69, 71], "transform": [2, 3, 4, 7, 10, 12, 13, 14, 15, 18, 19, 20, 21, 22, 23, 25, 26, 27, 28, 29, 31, 32, 33, 34, 38, 39, 40, 66, 69, 70], "jett": 2, "janiak": 2, "toi": [2, 14], "univers": 2, "icml": 2, "bilal": 2, "chughtai": 2, "n2g": 2, "scalabl": 2, "approach": [2, 5, 10, 70, 71], "quantifi": [2, 36], "larg": [2, 5, 7, 15, 18, 38, 69, 71, 72, 73], "languag": [2, 12, 14, 35, 63, 69, 70, 71], "workshop": 2, "rtml": 2, "alex": [2, 71], "foot": [2, 14, 71], "esben": 2, "kran": 2, "ioanni": 2, "konsta": 2, "fazl": 2, "barez": 2, "elicit": 2, "latent": 2, "predict": [2, 7, 10, 12, 14, 22, 35, 36, 69, 70, 71], "tune": [2, 12, 13, 38, 69, 71, 72], "len": [2, 10, 38], "nora": 2, "belros": 2, "zach": 2, "furman": 2, "logan": 2, "danni": 2, "halawi": 2, "igor": 2, "ostrovski": 2, "lev": 2, "mckinnei": 2, "stella": 2, "biderman": 2, "contribut": [2, 5, 10, 14, 70], "being": [2, 5, 10, 12, 13, 14, 15, 36, 37, 40, 69, 70, 71], "induct": [2, 4, 35, 36, 38], "head": [2, 4, 7, 10, 12, 13, 14, 15, 16, 18, 22, 24, 35, 36, 38, 40, 67, 69], "phase": 2, "replic": [2, 4, 14, 16, 35, 70, 71], "partial": [2, 70, 71], "context": [2, 10, 14, 37, 40, 69, 70, 71], "learn": [2, 3, 7, 15, 63, 69, 70, 71, 73], "connor": 2, "kissan": 2, "decis": [2, 3], "script": [2, 7], "train": [2, 7, 8, 9, 10, 12, 13, 14, 15, 35, 38, 69, 70, 73], "which": [2, 3, 5, 6, 7, 10, 12, 13, 14, 15, 35, 36, 37, 38, 39, 40, 69, 70, 71], "intermedi": [2, 10, 14, 37, 71], "activ": [2, 3, 4, 5, 7, 10, 12, 13, 14, 15, 16, 37, 40, 65, 69, 73], "perform": [2, 6, 7, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 35, 37, 69, 70, 71], "attribut": [2, 4, 7, 10, 18, 40, 69, 71], "ablat": [2, 70, 71], "up": [2, 3, 4, 5, 10, 14, 15, 37, 40, 63, 69, 70, 71], "initi": [2, 5, 8, 14, 15, 24, 37, 68, 69, 70, 71], "work": [2, 3, 4, 5, 7, 10, 12, 13, 14, 18, 37, 38, 69, 70, 71, 73], "found": [2, 3, 5, 6, 14, 15, 70, 71], "demo": [3, 5, 16, 38, 72], "how": [3, 5, 7, 10, 14, 36, 40, 63, 70, 71, 73], "basic": [3, 7, 14, 35, 69, 70], "To": [3, 4, 5, 10, 14, 15, 18, 37, 38, 70, 71], "see": [3, 5, 7, 10, 12, 13, 14, 15, 18, 24, 32, 33, 36, 38, 40, 69, 70, 71, 73], "exploratori": [3, 7, 36, 69, 71, 73], "analysi": [3, 7, 10, 14, 36, 69, 71, 73], "practic": [3, 4, 7, 70, 71], "look": [3, 4, 5, 7, 8, 10, 14, 18, 36, 40, 70, 71], "out": [3, 5, 7, 10, 14, 16, 40, 69, 70, 71], "my": [3, 5, 7, 14, 15, 69, 70, 71, 73], "analys": [3, 7, 10, 14, 71], "indirect": [3, 4, 7, 35], "object": [3, 4, 5, 7, 10, 11, 12, 13, 14, 15, 16, 35, 37, 38, 39, 63, 69], "identif": [3, 4, 7, 35], "record": [3, 7, 71], "myself": [3, 5, 7, 71], "veri": [3, 4, 5, 7, 10, 15, 16, 35, 69, 70, 71, 73], "young": [3, 4, 71], "small": [3, 4, 5, 6, 7, 10, 14, 15, 35, 38, 69, 70, 71, 72, 73], "field": [3, 4, 5, 14, 69, 71, 73], "lot": [3, 4, 5, 7, 10, 11, 39, 40, 69, 70, 71, 73], "open": [3, 4, 5, 14, 35, 73], "problem": [3, 4, 5, 71, 73], "would": [3, 4, 5, 12, 18, 32, 70, 71, 73], "help": [3, 4, 5, 15, 40, 70, 71], "try": [3, 4, 10, 14, 36, 70, 71], "concret": [3, 4, 70, 71], "figur": [3, 40, 70, 71], "where": [3, 5, 6, 10, 11, 12, 13, 14, 15, 18, 33, 36, 37, 38, 40, 63, 69, 70, 71], "skill": [3, 71], "kei": [3, 4, 5, 10, 12, 13, 14, 15, 18, 19, 24, 30, 31, 33, 35, 36, 39, 40, 69, 70, 71], "resourc": [3, 4, 5], "new": [3, 7, 10, 14, 37, 38, 39, 69, 70, 71], "tutori": [3, 4, 5, 70, 71], "scratch": [3, 4, 70], "an": [3, 4, 7, 10, 11, 12, 13, 14, 15, 18, 35, 36, 37, 38, 39, 40, 63, 67, 69, 70, 73], "accompani": [3, 4, 7, 71], "templat": [3, 35], "yourself": [3, 14, 70, 71], "One": [3, 5, 14, 70, 71, 73], "signific": [3, 37, 70, 71], "design": [3, 5, 10, 70, 71, 73], "made": [3, 5, 35, 70, 71], "wa": [3, 5, 6, 10, 12, 13, 14, 15, 35, 40, 70, 71], "singl": [3, 5, 10, 12, 14, 18, 31, 32, 33, 39, 40, 69, 70, 71], "implement": [3, 5, 12, 13, 14, 18, 40, 66, 69, 70, 71], "could": [3, 5, 70, 71], "support": [3, 5, 7, 12, 13, 14, 15, 30, 36, 37, 65, 66, 69, 70, 71], "rang": [3, 4, 5, 14, 16, 36, 40, 69, 70, 71], "subtli": [3, 18], "differ": [3, 5, 6, 10, 12, 13, 14, 15, 18, 35, 36, 37, 40, 69, 70, 71], "style": [3, 5, 10, 12, 13, 14, 15, 19, 36, 70, 71, 73], "upsid": 3, "just": [3, 4, 5, 10, 14, 15, 35, 40, 69, 70, 71], "arbitrari": [3, 14, 70, 71], "name": [3, 5, 10, 14, 15, 35, 36, 37, 38, 40, 63, 69], "But": [3, 10, 14, 40, 69, 70, 71], "downsid": 3, "py": [3, 5, 12, 13, 66], "compon": [3, 8, 9, 10, 12, 13, 14, 15, 67, 69, 70, 71], "difficult": [3, 10], "recommend": [3, 8, 10, 14, 15, 16, 37, 70, 71], "clean": [3, 40, 69, 70, 71], "minim": [3, 5, 71], "intern": [3, 5, 10, 14, 40, 70, 71, 73], "architectur": [3, 12, 13, 70], "significantli": [3, 12, 13, 14, 35, 40, 70, 71], "clearer": 3, "better": [3, 14, 15, 35, 36, 38, 70, 71], "document": [3, 14, 69, 71], "pip": [3, 5, 70, 71], "git": 3, "import": [3, 5, 10, 14, 16, 35, 39, 40, 69, 73], "known": [3, 73], "easytransform": [3, 71, 73], "break": [3, 5, 10, 70, 71], "been": [3, 5, 10, 14, 69, 71], "sinc": [3, 5, 10, 14, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 70, 71], "renam": [3, 5], "old": [3, 5, 38, 71], "version": [3, 7, 14, 35, 37, 66, 70, 71], "legaci": [3, 36], "run": [3, 5, 6, 10, 14, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 38, 39, 40, 63, 70, 73], "v1": 3, "avail": [3, 5, 7, 10, 14, 15, 36, 38], "requir": [3, 5, 12, 13, 14, 40, 69], "luckili": 3, "provid": [3, 10, 12, 13, 14, 25, 28, 32, 37, 68, 69], "wai": [3, 5, 6, 10, 14, 15, 37, 69, 70, 71], "those": [3, 5, 6, 14, 37, 69, 70], "configur": [3, 5, 15, 63, 68], "environment": 3, "variabl": [3, 12, 13], "simpli": [3, 5, 70], "token": [3, 7, 10, 12, 13, 14, 15, 16, 18, 19, 21, 22, 23, 24, 27, 31, 32, 33, 35, 36, 38, 40, 69, 70], "hf_token": 3, "agreement": 3, "issu": [3, 5, 14, 70, 71], "attempt": [3, 5, 14], "ue": 3, "befor": [3, 10, 12, 13, 14, 15, 18, 19, 20, 37, 69, 70, 71], "relat": [3, 14, 18, 70, 71], "consol": 3, "output": [3, 5, 7, 10, 12, 13, 14, 15, 18, 36, 37, 40, 69, 70, 71], "point": [3, 5, 10, 12, 13, 14, 15, 16, 33, 37, 69, 70, 73], "As": [3, 14, 15, 69, 70, 71], "23": [3, 70, 71], "24": [3, 14, 69, 70, 71, 72], "current": [3, 5, 10, 12, 13, 14, 15, 18, 19, 24, 36, 70, 71], "co": [3, 69], "mistralai": [3, 38], "mixtral": [3, 5, 6, 38, 42, 72], "8x7b": [3, 38], "v0": [3, 38, 72], "mistral": [3, 15, 18, 35, 38, 42, 72], "7b": [3, 5, 7, 38, 71, 72], "instruct": [3, 38, 71, 72], "mean": [4, 5, 10, 14, 15, 16, 18, 19, 24, 28, 29, 36, 37, 69, 70, 71], "": [4, 5, 7, 10, 11, 12, 13, 14, 15, 16, 18, 35, 36, 37, 38, 40, 68, 69, 71, 73], "both": [4, 5, 10, 14, 15, 18, 36, 37, 39, 66, 70, 71], "low": [4, 11, 14, 18, 69, 71], "hang": [4, 71], "fruit": [4, 71], "bar": [4, 14], "entri": [4, 18, 39, 40, 71], "The": [4, 5, 7, 10, 11, 12, 13, 14, 15, 16, 18, 19, 22, 24, 27, 31, 32, 33, 35, 36, 37, 38, 39, 40, 63, 68, 69, 70, 71, 73], "standard": [4, 6, 14, 15, 70, 71], "answer": [4, 10, 40, 69, 70, 71], "why": [4, 5, 10, 18, 69, 70, 71], "yet": [4, 5, 12, 13, 14, 70, 71, 73], "aren": [4, 38, 71], "t": [4, 5, 10, 11, 12, 13, 14, 15, 16, 18, 35, 38, 69, 70, 71, 73], "enough": [4, 5, 10, 70, 71], "peopl": [4, 5, 71], "guid": [4, 71], "arena": 4, "callum": [4, 71], "mcdougal": [4, 71], "comprehens": [4, 71], "introduct": 4, "mech": [4, 70], "interp": [4, 70], "written": [4, 5, 7, 70], "snippet": 4, "copi": [4, 12, 13, 70], "come": [4, 5, 14, 15, 40, 70, 71], "exercis": [4, 70], "solut": [4, 5, 70, 71], "notabl": [4, 14, 37, 70, 71], "video": [4, 7, 70, 71], "me": [4, 5, 38, 71, 73], "good": [4, 5, 7, 10, 35, 69, 70, 71, 73], "cover": [4, 14, 71], "foundat": [4, 71], "concept": [4, 70, 71], "wild": [4, 10, 70, 71], "techniqu": [4, 7, 14, 40, 70, 71], "direct": [4, 5, 10, 14, 16, 37, 40, 71], "logit": [4, 5, 6, 7, 10, 12, 13, 14, 15, 22, 35, 40, 69, 71], "patch": [4, 5, 7, 8, 9], "paper": [4, 7, 10, 14, 15, 18, 32, 35, 40, 71], "read": [4, 5, 7, 10, 14, 71], "200": [4, 71], "explain": [4, 7, 70, 71], "jargon": 4, "unfamiliar": [4, 70], "term": [4, 10, 14, 70], "go": [4, 5, 7, 40, 71], "across": [4, 5, 10, 12, 13, 14, 16, 38, 40, 68, 70, 71], "youtub": 4, "channel": 4, "content": [4, 35, 70, 71], "walkthrough": [4, 70, 71], "am": 5, "happi": 5, "announc": 5, "now": [5, 7, 14, 15, 70, 71], "releas": 5, "recent": 5, "primari": 5, "motiv": [5, 70], "behind": [5, 18, 70], "jump": [5, 71], "transit": [5, 70], "strictli": [5, 10, 71], "describ": [5, 15, 69, 70], "At": [5, 70], "last": [5, 10, 14, 69, 71], "minut": 5, "did": [5, 10, 69, 70, 71], "remov": [5, 10, 11, 14, 18, 37, 69, 70, 71, 73], "hookedsa": 5, "had": [5, 70, 71], "saelen": 5, "bundl": [5, 71], "major": 5, "hand": [5, 70, 71], "modif": 5, "affect": [5, 14, 40, 70], "bryce": 5, "meyer": 5, "softwar": 5, "engin": [5, 40, 70, 71, 73], "littl": [5, 15, 71, 73], "under": [5, 10, 14, 24], "15": [5, 70, 71], "profession": [5, 7], "experi": [5, 7, 15, 70, 71, 73], "wide": 5, "expertis": 5, "embed": [5, 7, 10, 12, 13, 14, 15, 18, 21, 22, 27, 33, 70, 71], "comput": [5, 10, 11, 14, 20, 21, 22, 23, 25, 26, 28, 29, 30, 32, 34, 36, 37, 39, 40, 69, 70, 71, 73], "coupl": 5, "gotten": [5, 14], "ml": [5, 70, 71, 73], "especi": [5, 14, 70, 71], "ai": [5, 15, 18, 38, 69, 71], "safeti": 5, "nine": 5, "march": 5, "chat": [5, 38, 71, 72], "bit": [5, 15, 70, 71], "he": [5, 71], "ask": 5, "might": [5, 10, 12, 13, 70], "interest": [5, 7, 12, 13, 14, 70, 71], "take": [5, 7, 10, 14, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 40, 69, 70, 71, 73], "role": 5, "maintain": 5, "basi": [5, 12, 14], "april": 5, "far": [5, 14, 69, 70, 71], "pretti": [5, 10, 14, 69, 70, 71], "mani": [5, 14, 24, 39, 40, 63, 70, 71], "kind": [5, 10, 70, 71], "address": 5, "everi": [5, 10, 12, 13, 14, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 40, 63, 69, 70, 71], "await": 5, "repli": 5, "total": [5, 70, 71], "around": [5, 6, 10, 12, 13, 14, 37, 40, 68, 70, 71], "30": [5, 70, 71, 72], "20": [5, 69, 70, 71, 72], "pr": 5, "were": [5, 10, 14, 15, 35, 69, 70, 71, 73], "limit": [5, 12, 13, 14, 70], "llama": [5, 7, 15, 38, 42, 72], "quantiz": [5, 15], "hookedsaetransform": 5, "brand": 5, "class": [5, 10, 11, 12, 13, 14, 15, 16, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 37, 38, 39, 63, 69], "splice": 5, "autoencod": 5, "two": [5, 6, 11, 12, 32, 36, 38, 40, 69, 70, 71], "goal": [5, 70, 71, 73], "posit": [5, 7, 10, 12, 13, 14, 15, 18, 21, 27, 30, 33, 35, 36, 37, 38, 40, 69, 70, 71], "while": [5, 12, 13, 14, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 70, 71], "remain": [5, 10, 14, 37, 71], "power": [5, 71], "who": 5, "push": 5, "second": [5, 14, 35, 70, 71], "base": [5, 7, 10, 11, 12, 13, 14, 15, 16, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 37, 38, 39, 63, 68, 69, 70, 71, 72], "easier": [5, 14, 35, 70, 71, 73], "futur": 5, "llm": 5, "continu": [5, 40, 70, 71], "acceler": [5, 70, 71], "feel": [5, 70, 71, 73], "massiv": [5, 70, 71], "amount": [5, 10, 15, 70], "momentum": [5, 63], "moment": [5, 18, 19, 24], "hope": [5, 71], "carri": 5, "over": [5, 10, 14, 40, 69, 70, 71], "background": [5, 70], "know": [5, 7, 10, 12, 13, 70, 71], "talk": [5, 71], "ensur": [5, 14, 70], "meet": 5, "person": [5, 70], "spoken": 5, "dozen": 5, "commun": 5, "happen": [5, 70, 71], "appoint": 5, "curiou": 5, "hear": 5, "anyon": [5, 35], "tool": [5, 7, 71, 73], "absolut": [5, 6, 12, 13, 14, 15, 18, 27, 36, 69, 70, 71], "beginn": 5, "complet": [5, 10, 18, 70, 71], "expert": [5, 15], "Not": [5, 18, 19, 24, 26], "idea": [5, 14, 18, 40, 70, 71, 73], "evolv": 5, "biggest": [5, 10], "previous": [5, 70], "offici": [5, 38, 71], "instanc": [5, 12, 13, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 36, 37, 70], "compat": [5, 14, 38], "through": [5, 14, 70, 71], "forward": [5, 10, 12, 13, 14, 15, 18, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 37, 39, 71], "start": [5, 10, 13, 14, 15, 18, 40, 69, 70, 71], "todai": [5, 71, 73], "assur": 5, "abl": [5, 70, 71], "upgrad": 5, "worri": 5, "There": [5, 6, 10, 12, 13, 36, 38, 69, 70, 71, 73], "right": [5, 12, 13, 14, 18, 19, 40, 69, 70, 71], "move_model": [5, 10], "activationcach": [5, 8, 9, 12, 13, 14, 36, 40, 69, 70, 71], "function": [5, 10, 12, 13, 14, 15, 18, 20, 21, 22, 23, 24, 25, 26, 28, 29, 32, 34, 36, 37, 38, 40, 63, 65, 66, 68, 69, 71, 73], "cache_al": [5, 37], "hook_point": [5, 8, 9, 14, 71], "keep": [5, 10, 14, 37, 70, 71, 73], "thing": [5, 11, 14, 15, 18, 40, 70, 71, 73], "simpl": [5, 70, 71], "howev": [5, 10, 14, 24, 35, 70, 71], "them": [5, 10, 14, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 40, 69, 70, 71], "adapt": [5, 71], "awai": [5, 14, 70], "along": [5, 14, 18, 69, 70, 71], "anyth": [5, 35, 70], "mark": [5, 70, 71], "whenev": [5, 37], "someth": [5, 14, 35, 70], "becom": [5, 70, 71], "promin": 5, "sort": [5, 71], "slip": 5, "scenario": 5, "situat": 5, "handl": [5, 14, 37], "persist": [5, 37], "allow": [5, 14, 36, 40, 69, 70, 71], "reli": 5, "interrupt": 5, "still": [5, 37, 70], "encourag": [5, 18, 69], "everyon": 5, "period": [5, 70, 71], "ey": 5, "don": [5, 10, 12, 13, 14, 15, 16, 35, 69, 70, 71, 73], "imagin": [5, 70], "often": [5, 10, 14, 15, 38, 69, 70, 71], "save": [5, 10, 14, 15, 36, 63, 69, 70, 71], "troubl": [5, 10], "move": [5, 10, 12, 13, 14, 40, 70, 71], "three": [5, 14, 40, 69, 70], "timefram": 5, "plan": 5, "state": [5, 10, 14, 31, 37, 70, 71, 73], "tracker": 5, "categor": 5, "easi": [5, 10, 14, 69, 70, 71, 73], "date": [5, 14], "below": [5, 12, 13, 14, 70], "draft": 5, "our": [5, 33, 70, 71], "priorit": 5, "feedback": [5, 69, 70, 71, 73], "surfac": 5, "other": [5, 10, 12, 14, 18, 19, 24, 36, 37, 38, 40, 70], "improv": [5, 69, 70, 71], "achiev": [5, 10, 71], "diagnos": 5, "variou": [5, 15, 37, 70, 71], "area": 5, "memori": [5, 10, 11, 12, 13, 14, 15, 70, 71], "leak": 5, "occur": [5, 70], "seem": [5, 14, 15, 35, 38, 70, 71], "refer": [5, 10, 14, 19, 37, 70, 71], "properli": [5, 35, 70], "thu": 5, "caus": 5, "garbag": 5, "collect": 5, "correctli": [5, 14], "identifi": [5, 7, 14, 40, 70, 71], "proper": 5, "overal": [5, 70], "deal": [5, 14, 18, 69, 70], "larger": [5, 10, 35, 70, 71], "task": [5, 7, 12, 14, 15, 35, 40, 63, 70], "explor": [5, 69, 71], "abil": [5, 10, 40, 70], "batch": [5, 10, 12, 13, 14, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 31, 32, 33, 34, 35, 37, 39, 40, 63, 66, 67, 69, 70, 71], "process": [5, 14, 15, 38, 69, 70, 71], "alreadi": [5, 10, 14, 69, 70, 71], "share": 5, "well": [5, 7, 14, 16, 37, 40, 69, 70, 71], "togeth": [5, 14, 69, 70, 71], "separ": [5, 12, 13, 14, 15, 37, 67, 69, 70, 71], "volunt": 5, "said": [5, 71], "submiss": 5, "discuss": [5, 70], "think": [5, 14, 35, 69, 70, 71], "few": [5, 10, 12, 13, 70, 71], "week": 5, "confus": [5, 70, 71], "among": 5, "calcul": [5, 10, 11, 12, 13, 14, 15, 18, 24, 36, 67, 69, 70, 71], "match": [5, 10, 14, 36, 66, 70], "huggingfac": [5, 12, 13, 14, 15, 35, 38, 66, 69, 71, 73], "solv": [5, 37, 70, 71, 73], "systemat": 5, "submit": 5, "show": [5, 7, 14, 16, 36, 70, 71, 73], "order": [5, 18, 40, 69, 70], "allevi": 5, "build": [5, 37, 71, 73], "spit": 5, "tabl": [5, 70, 71], "u": [5, 11, 14, 40, 69, 70, 71], "snapshot": 5, "store": [5, 10, 14, 15, 24, 37, 39, 40, 63, 70, 71], "repo": [5, 70], "regener": 5, "cumul": [5, 14, 69], "valu": [5, 7, 10, 11, 12, 13, 14, 15, 18, 24, 30, 31, 33, 36, 38, 39, 40, 69, 70, 71, 73], "creat": [5, 7, 10, 14, 18, 37, 70, 71], "robust": [5, 10, 70], "big": [5, 36, 38, 69, 70, 71], "famili": [5, 15, 71], "hard": [5, 14, 70, 71], "even": [5, 7, 11, 14, 15, 18, 35, 38, 70, 71, 73], "smallest": [5, 71], "thought": [5, 10, 70, 71], "thrown": 5, "topic": [5, 7], "best": [5, 7, 14, 70, 71], "guess": [5, 70], "reason": [5, 18, 19, 24, 70, 71], "untrain": 5, "eg": [5, 10, 14, 35, 40, 69, 70, 71], "randomli": [5, 14, 15, 71], "weight": [5, 7, 12, 13, 14, 15, 18, 24, 26, 63, 66, 69, 70, 71, 73], "verifi": [5, 7, 70, 71], "load": [5, 10, 12, 13, 14, 15, 35, 38, 69, 70, 73], "result": [5, 10, 12, 13, 14, 15, 21, 30, 33, 36, 38, 40, 69, 70, 71, 73], "accur": 5, "sens": [5, 11, 37, 70, 71], "consist": [5, 14, 70, 71], "sampl": [5, 14, 35, 69], "size": [5, 10, 14, 15, 18, 35, 63, 69, 70, 71], "against": [5, 70], "bite": 5, "success": [5, 70], "turn": [5, 10, 14, 69, 70, 71], "effici": [5, 11, 18, 69, 71], "proof": [5, 70], "put": [5, 70, 71], "strong": 5, "opinion": 5, "most": [5, 10, 14, 37, 69, 70, 71, 73], "roundtabl": 5, "wrapper": [5, 10, 12, 13, 14, 40, 68, 71], "plugin": 5, "addit": [5, 7, 12, 13, 14, 70], "outsid": 5, "publish": 5, "themselv": [5, 14], "final": [5, 6, 10, 12, 13, 14, 15, 18, 69, 70, 71], "overhaul": 5, "composit": [5, 14, 70, 71], "util": [5, 8, 9, 10, 11, 14, 36, 37, 38, 63, 70, 71], "isol": [5, 70], "rapidli": 5, "itself": [5, 36, 69, 70], "none": [5, 10, 12, 13, 14, 15, 16, 18, 19, 20, 21, 24, 25, 27, 28, 30, 31, 33, 35, 36, 37, 38, 40, 63, 68, 69, 70, 71], "pain": [5, 71], "rel": [5, 15, 30, 70], "grow": 5, "exponenti": 5, "whole": [5, 70, 71], "explod": 5, "section": [5, 70, 71], "relev": [5, 14, 15, 18, 40, 69, 70, 71], "skip": [5, 14, 70, 71], "setup": [5, 14, 37, 40], "act": [5, 37, 40, 69, 70, 71], "vast": 5, "due": [5, 6, 14, 71], "potenti": 5, "mismatch": [5, 36], "between": [5, 10, 14, 15, 18, 36, 40, 69, 70, 71, 73], "meant": 5, "repres": [5, 11, 12, 15, 32, 36, 40, 69, 70, 71], "updat": [5, 7, 14, 39, 40, 68, 70, 71], "readi": 5, "sent": 5, "justifi": 5, "bug": [5, 7, 10, 15, 69, 71], "fix": [5, 37, 70, 71], "exist": [5, 36, 70, 71], "split": [5, 14, 18, 38, 69, 70, 71], "group": [5, 15, 18, 24], "call": [5, 10, 12, 13, 14, 15, 20, 21, 22, 23, 24, 25, 26, 28, 29, 32, 34, 37, 38, 69, 70, 71], "again": [5, 70], "piec": [5, 70], "tradition": 5, "everyth": [5, 37, 40, 69, 71], "That": [5, 70], "mock": 5, "spi": 5, "control": [5, 15, 40, 70, 71], "input": [5, 10, 12, 13, 14, 15, 24, 27, 32, 37, 38, 39, 40, 66, 67, 69, 70, 71], "side": [5, 11, 14], "effect": [5, 10, 14, 15, 40, 70, 71], "certain": [5, 15, 40], "logic": 5, "entir": [5, 10, 14, 40, 70], "rule": 5, "incredibli": [5, 71], "cannot": [5, 14, 69, 71], "origin": [5, 15, 16, 18, 70, 71], "pass": [5, 10, 14, 15, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 32, 33, 34, 36, 37, 38, 39, 69, 71], "bunch": [5, 10, 12, 13, 14, 37, 70, 71], "Being": [5, 70], "live": [5, 12, 13, 14, 70, 71], "ci": 5, "report": 5, "great": [5, 7, 71, 73], "get": [5, 7, 10, 12, 13, 14, 15, 16, 18, 35, 36, 37, 40, 68, 70, 71, 73], "review": 5, "meaning": [5, 15, 40], "than": [5, 6, 10, 14, 15, 18, 20, 35, 36, 37, 40, 69, 70, 71], "ever": [5, 69], "substanti": 5, "let": [5, 15, 69, 70, 71, 73], "shift": [5, 40, 71], "favor": 5, "individu": [5, 7, 10, 14, 18, 70], "3000": 5, "18": [5, 69, 70, 71, 72], "distinct": 5, "either": [5, 14, 36, 37, 38, 40, 70, 71], "interdepend": 5, "least": [5, 40, 71], "depend": [5, 10, 14, 71], "its": [5, 10, 14, 18, 37, 70, 71, 73], "own": [5, 7, 70, 71], "realli": [5, 35, 36, 69, 70, 71], "anywher": [5, 40], "els": [5, 14, 15, 18, 38, 69, 70, 71], "mlp": [5, 10, 12, 13, 14, 15, 20, 33, 40, 69, 70, 71], "exactli": [5, 14, 38, 66, 70, 71], "thank": [5, 71], "excit": 5, "standpoint": 5, "perspect": [5, 71], "comparison": [5, 36], "worth": [5, 7, 10, 70, 71], "enabl": [5, 10, 14, 70, 71, 73], "huge": 5, "impact": [5, 10], "bring": 5, "realiti": [5, 70], "semver": 5, "older": [5, 71], "log": [5, 14, 63, 69, 70, 71], "data": [5, 6, 7, 14, 35, 69, 70, 71], "expos": [5, 14, 73], "properti": [5, 11, 12, 13, 14, 18, 24, 69, 70, 71], "minor": [5, 70], "bump": [5, 71], "whatsoev": 5, "With": [5, 18, 71], "fact": [5, 14, 70, 71, 73], "discov": 5, "earlier": [5, 70, 71], "extent": 5, "probabl": [5, 7, 14, 35, 36, 40, 69, 70, 71], "regardless": [5, 69], "stand": 5, "reliabl": 5, "17": [5, 70, 71], "possibli": 5, "easiest": [5, 70], "fresh": 5, "consum": [5, 10, 11], "top": [6, 14, 69, 70, 71], "k": [6, 10, 11, 12, 13, 14, 16, 18, 24, 40, 69, 70, 71], "gate": [6, 14], "hidden": [6, 15, 31, 71], "amplifi": 6, "greatli": [6, 71, 73], "select": [6, 10, 69, 70, 71], "lead": [6, 7, 12, 13, 37, 69, 71], "higher": [6, 14, 70], "normal": [6, 10, 14, 15, 26, 69, 70, 71, 73], "varianc": [6, 70], "test": [6, 7, 14, 35, 36, 69, 70, 71], "half": [6, 11, 12, 13, 14, 18, 35, 71], "precis": [6, 36, 40, 70, 71], "deviat": [6, 15, 71], "compar": [6, 35, 71, 73], "2e": 6, "mitig": 6, "disabl": [6, 14, 36, 37, 70], "preprocess": [6, 12, 13, 70], "option": [6, 10, 12, 13, 14, 15, 16, 18, 19, 20, 21, 24, 25, 27, 28, 30, 31, 33, 35, 36, 37, 38, 39, 40, 63, 68, 69], "from_pretrained_no_process": [6, 14], "increas": [6, 40, 70, 71], "colab": [7, 70, 71, 73], "blob": [7, 66], "ipynb": 7, "causal": [7, 12, 15, 40, 70, 71], "intervent": [7, 40, 70, 71], "matter": [7, 14, 40, 70, 71], "produc": [7, 14, 40, 70], "incomplet": 7, "gradient": [7, 10, 37, 63, 71], "approxim": [7, 70, 71], "bad": [7, 14], "residu": [7, 10, 12, 13, 14, 15, 18, 31, 33, 34, 40, 71], "stream": [7, 10, 12, 13, 14, 15, 31, 33, 40, 69, 71], "after": [7, 10, 14, 15, 20, 26, 37, 63, 70, 71, 73], "demonstr": [7, 16, 70, 71], "focus": [7, 70, 71], "less": [7, 14, 18, 70], "rigor": [7, 70, 71], "grasp": 7, "steal": 7, "liber": [7, 37], "phenomenon": 7, "memoris": 7, "minimis": 7, "loss": [7, 10, 14, 35, 37, 40, 63, 69, 70, 71], "longer": 7, "generalis": [7, 70, 71], "sharp": [7, 71], "decreas": [7, 18, 69, 70], "modular": [7, 69], "grok": 7, "light": 7, "explan": [7, 40, 70], "ll": [7, 14, 36, 70, 71], "pair": [7, 11, 14, 18, 36, 69, 70, 71], "seri": [7, 10, 71], "detector": [7, 36], "detect": [7, 36, 70, 71], "sever": [7, 10, 14, 69, 70, 71], "custom": [7, 14, 15, 21, 35, 37, 69, 70, 71], "algorithm": [7, 11, 15, 71, 73], "interact": [7, 65, 70, 71], "neuroscop": [7, 71], "hacki": [7, 69], "web": [7, 71], "visualis": [7, 70], "front": 7, "visual": [7, 12, 13, 71], "dynam": [7, 15, 71], "convert": [7, 12, 13, 14, 38, 69, 70, 71], "meta": [7, 14, 38, 69, 70, 71, 72], "until": [7, 10, 14, 37, 70, 71], "multi": [7, 10, 69, 71], "gpu": [7, 10, 11, 12, 13, 14, 70, 71], "access": [7, 10, 15, 37, 69, 70], "No": [7, 71], "previou": [7, 10, 14, 31, 33, 36, 70, 71], "port": 7, "excel": [7, 10, 70, 71, 73], "sequenc": [7, 12, 13, 14, 15, 18, 32, 35, 36, 37, 40, 69, 70, 71], "investig": [7, 10, 14, 36, 70, 71], "svd": [7, 11, 14, 16, 71], "conjectur": 7, "post": [7, 10, 15, 16, 70, 71], "singular": [7, 11, 14, 16, 71], "decomposit": [7, 10, 11, 14, 70, 71], "matric": [7, 11, 12, 13, 14, 16, 18, 19, 24, 66, 70, 71], "surprisingli": 7, "reproduc": [7, 15, 36], "further": [7, 10, 14, 69, 70, 71], "tracr": 7, "cool": 7, "deepmind": 7, "compil": 7, "program": [7, 71, 73], "rasp": 7, "jax": 7, "form": [7, 10, 11, 14, 40, 70, 71], "pytorch": [7, 14, 15, 35, 37, 71], "brows": 8, "first": [8, 10, 14, 15, 35, 38, 40, 69, 70, 71], "submodul": 8, "factoredmatrix": [8, 9, 12, 13, 18, 69, 71], "hookedencod": [8, 9, 68], "hookedencoderdecod": [8, 9, 68], "hookedtransformerconfig": [8, 9, 14, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 38, 39, 44, 45, 51, 52, 58, 59, 60, 61, 62, 68], "svdinterpret": [8, 9], "eval": [8, 9, 71], "head_detector": [8, 9], "past_key_value_cach": [8, 9], "subpackag": 8, "pretrain": [8, 9, 10, 12, 13, 14, 15, 35, 38, 69, 70, 71], "core": [10, 14, 70, 71, 73], "varieti": [10, 71], "helper": [10, 14, 18, 35, 37, 40, 69, 71], "skim": 10, "method": [10, 12, 13, 14, 15, 37, 38, 39, 69, 70, 71], "back": [10, 15, 18, 19, 24, 71], "cache_dict": 10, "dict": [10, 12, 13, 14, 15, 18, 19, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 32, 33, 34, 35, 36, 37, 38, 69], "str": [10, 12, 13, 14, 15, 18, 19, 24, 30, 35, 36, 37, 38, 39, 40, 63, 68, 69, 70, 71], "tensor": [10, 11, 12, 13, 14, 16, 18, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 36, 37, 39, 40, 66, 67, 69, 70, 71], "has_batch_dim": 10, "bool": [10, 12, 13, 14, 15, 30, 31, 35, 36, 37, 38, 39, 40, 63, 69], "run_with_cach": [10, 12, 13, 14, 37, 70, 71], "particular": [10, 16, 36, 70, 71], "behaviour": [10, 14, 70, 71], "modal": 10, "step": [10, 14, 15, 38, 63, 69, 70, 71], "respons": [10, 15, 70], "prompt": [10, 14, 18, 35, 40, 69, 70, 71], "chicken": 10, "road": [10, 69], "specif": [10, 12, 13, 14, 18, 36, 40, 70, 71], "sublay": 10, "commonli": 10, "fall": 10, "categori": [10, 70], "dla": 10, "_logit": 10, "residual_stream": 10, "label": [10, 14, 15, 18, 19, 24, 38, 70, 71], "decompose_resid": [10, 70], "return_label": [10, 70], "0": [10, 12, 13, 14, 15, 16, 18, 27, 32, 36, 38, 63, 69, 70, 71, 72], "emb": [10, 17, 21, 27, 32, 69, 71], "pos_emb": [10, 15, 17, 71], "0_attn_out": 10, "proceed": 10, "space": [10, 12, 14, 69, 70, 71], "logit_attr": 10, "shape": [10, 12, 14, 18, 19, 24, 31, 32, 33, 40, 69, 70, 71], "torch": [10, 12, 13, 14, 15, 18, 24, 27, 31, 33, 37, 38, 40, 66, 68, 69, 70, 71], "10": [10, 14, 16, 69, 70, 71, 72], "7": [10, 35, 70, 71, 72], "most_important_component_idx": 10, "argmax": [10, 70], "3_attn_out": 10, "dig": [10, 70, 71, 73], "granular": 10, "get_full_resid_decomposit": 10, "stack": [10, 12, 13, 14, 40, 69, 70, 71], "equal": [10, 15], "struggl": 10, "construct": [10, 12, 13], "joke": 10, "trivial": 10, "accumulated_resid": [10, 70], "footgun": [10, 37], "sourc": [10, 14, 15, 18, 35, 40, 73], "track": [10, 70], "index": [10, 12, 13, 14, 15, 16, 18, 19, 24, 38, 40, 68, 69, 70, 71], "dimens": [10, 14, 15, 18, 24, 25, 28, 37, 40, 66, 67, 69, 70, 71], "vector": [10, 11, 14, 16, 18, 40, 70, 71], "q": [10, 12, 13, 14, 18, 24, 40], "z": [10, 14, 24, 36, 40, 70, 71], "po": [10, 12, 13, 14, 18, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 40, 67, 69, 70, 71], "head_index": [10, 14, 16, 18, 19, 24, 25, 26, 30, 31, 40, 67, 70, 71], "d_head": [10, 12, 13, 14, 15, 18, 19, 24, 38, 39, 67, 69, 70, 71, 72], "pattern": [10, 12, 13, 14, 18, 19, 24, 36, 40, 71], "softmax": [10, 14, 15, 18, 19, 24, 26, 69, 71], "attn_scor": [10, 15, 18, 19], "pre": [10, 14, 15, 18, 19, 24, 26, 29, 36, 69], "query_po": [10, 18, 19, 24, 70], "key_po": [10, 18, 19, 24, 70], "d_model": [10, 12, 13, 14, 15, 18, 19, 20, 22, 23, 24, 25, 26, 27, 28, 31, 33, 34, 38, 67, 69, 70, 71, 72], "mid": [10, 70], "solu_ln": [10, 15], "layernorm": [10, 12, 13, 14, 15, 20, 25, 26, 28, 69, 70], "d_mlp": [10, 12, 13, 14, 15, 26, 38, 69, 71, 72], "resid_pr": [10, 15, 20, 31, 33, 40, 70, 71], "resid_mid": [10, 40], "resid_post": [10, 15, 70], "attn_out": [10, 14, 15, 40, 70], "mlp_out": [10, 14, 15, 40, 70], "ln": [10, 14, 15, 70, 71], "lnpre": [10, 15], "scale": [10, 14, 15, 18, 19, 24, 69, 70, 71], "sometim": [10, 35, 70], "miss": [10, 70], "becaus": [10, 11, 12, 13, 14, 15, 18, 35, 69, 70, 71, 73], "appli": [10, 14, 15, 18, 20, 33, 37, 40, 69, 70, 71], "remove_batch_dim": [10, 37, 69, 71], "batch_siz": [10, 12, 32, 35, 37, 39, 63, 70, 71], "annot": [10, 71], "layers_cov": 10, "queri": [10, 12, 13, 14, 15, 18, 19, 24, 30, 36, 40, 71], "batch_and_pos_dim": 10, "ve": [10, 14, 18, 35, 37, 70, 73], "slice": [10, 37, 69, 70], "dictionari": [10, 12, 13, 14, 15, 36, 37, 38, 69, 71], "whether": [10, 12, 14, 15, 18, 32, 35, 37, 38, 40, 63, 69, 70, 71], "int": [10, 11, 12, 13, 14, 15, 16, 18, 19, 21, 23, 24, 25, 27, 28, 30, 31, 32, 33, 35, 36, 37, 38, 39, 40, 63, 68, 69, 70, 71], "incl_mid": [10, 70], "fals": [10, 12, 13, 14, 15, 30, 35, 36, 37, 38, 39, 40, 63, 69, 70, 71], "apply_ln": [10, 70], "pos_slic": [10, 37, 70], "union": [10, 12, 13, 14, 15, 16, 18, 19, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 32, 33, 34, 36, 37, 38, 39, 40, 68, 69, 70], "tupl": [10, 11, 12, 13, 14, 18, 24, 36, 37, 40, 69, 71], "ndarrai": [10, 14, 37, 69], "mlp_input": [10, 14], "float": [10, 11, 12, 13, 14, 15, 18, 20, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 33, 34, 36, 38, 39, 40, 63, 66, 67, 69, 70, 71], "accumul": [10, 14, 70], "sub": [10, 71], "www": 10, "lesswrong": 10, "ackrb8wdpdan6v6ru": 10, "believ": [10, 35, 70], "vocabulari": [10, 15, 70, 71], "rememb": 10, "norm": [10, 11, 14, 15, 21, 25, 26, 28, 29, 38, 63, 70, 71], "decod": [10, 13, 15, 31], "therefor": [10, 14, 69], "multipli": [10, 14, 18, 19, 36, 66, 70, 71], "unembed": [10, 12, 13, 14, 70, 71], "matrix": [10, 11, 12, 13, 14, 15, 16, 18, 24, 36, 69, 70], "w_u": [10, 12, 13, 14, 70, 71], "broken": [10, 37, 69, 70, 71], "down": [10, 14, 18, 19, 24, 70, 71], "einop": [10, 70, 71], "einsum": [10, 70, 71], "panda": [10, 40], "pd": [10, 40], "devic": [10, 12, 13, 14, 15, 16, 18, 30, 35, 37, 38, 39, 63, 64, 70, 71], "answer_token": [10, 70], "to_single_token": [10, 14, 70, 71], "2975": 10, "accum_resid": 10, "last_token_accum": 10, "9": [10, 69, 70, 71, 72], "64": [10, 38, 69, 71, 72], "50257": [10, 38, 71, 72], "layers_unembed": 10, "d_vocab": [10, 12, 13, 14, 15, 16, 38, 40, 69, 71, 72], "rank": [10, 11, 14, 18, 69, 70, 71], "correct": [10, 40, 68, 69, 70, 71], "sorted_indic": 10, "argsort": 10, "dim": [10, 14, 69, 70, 71], "descend": [10, 71], "rank_answ": 10, "nonzero": 10, "as_tupl": 10, "0_pre": 10, "4442": 10, "1_pre": [10, 70], "382": 10, "2_pre": 10, "982": 10, "3_pre": 10, "1160": 10, "4_pre": 10, "408": 10, "5_pre": 10, "145": 10, "6_pre": 10, "78": 10, "7_pre": 10, "387": 10, "final_post": 10, "6": [10, 14, 38, 69, 70, 71, 72], "dtype": [10, 12, 13, 14, 15, 18, 38, 68, 70, 71], "int64": [10, 69], "exclud": [10, 36], "n_layer": [10, 12, 13, 14, 15, 38, 40, 68, 70, 71, 72], "immedi": [10, 19, 69, 70, 71], "indic": [10, 12, 13, 32, 37, 40, 69, 70, 71], "taken": [10, 14, 71], "l": [10, 12, 13, 14, 70, 71], "noth": [10, 12, 13, 14, 37, 69, 70, 71], "essenti": [10, 14, 70, 71, 73], "rather": [10, 14, 15, 20, 40, 69, 70, 71], "graph": [10, 70, 71], "apply_ln_to_stack": [10, 14, 69, 70], "residual_stack": [10, 70], "num_compon": 10, "batch_slic": 10, "batch_and_pos_dims_out": 10, "treat": [10, 14, 15, 70, 71], "factor": [10, 11, 14, 15, 70], "simul": [10, 14, 70, 71], "global": [10, 18, 19, 24, 30, 37, 69, 70, 71], "element": [10, 14, 18, 36, 40, 69, 71], "rmsnorm": [10, 15, 28], "unchang": [10, 12, 13, 14, 69, 70, 71], "whose": [10, 12, 13, 14, 35, 69, 70], "trail": [10, 11, 69], "assum": [10, 12, 13, 14, 15, 25, 28, 32, 37, 40, 63, 69, 70], "hook_scal": [10, 69, 70, 71], "unemb": [10, 14, 15, 17, 70, 71], "map": [10, 12, 13, 14, 15, 18, 36, 37, 70, 71], "ie": [10, 12, 13, 14, 15, 18, 38, 40, 69, 70, 71], "ln2": [10, 33, 69, 71], "ln1": [10, 15, 33, 69, 71], "ln_final": [10, 14, 70, 71], "apply_slice_to_batch_dim": 10, "compute_head_result": 10, "sum": [10, 11, 14, 15, 21, 36, 69, 70, 71], "plu": 10, "b_o": [10, 12, 13, 14, 71], "intend": [10, 15, 69], "use_attn_result": [10, 14, 15], "forget": 10, "liter": [10, 12, 13, 14, 16, 36, 37, 40], "incl_emb": 10, "decompos": 10, "incl": 10, "expand_neuron": 10, "bias": [10, 12, 13, 14, 15, 63, 70], "expand": [10, 14, 24], "get_neuron_result": 10, "neuron_slic": 10, "num_neuron": 10, "subset": [10, 15, 35, 70, 71], "specifi": [10, 12, 13, 14, 15, 24, 35, 36, 37, 68, 69, 71], "expens": [10, 11], "cheap": 10, "hook_emb": [10, 69, 71], "hook_pos_emb": [10, 71], "block": [10, 14, 15, 18, 19, 20, 24, 31, 33, 40, 69, 70, 71], "hook_resid_pr": [10, 71], "incorrect_token": [10, 70], "typic": [10, 12, 14, 32, 36, 70, 71], "revers": [10, 11, 40, 69, 70, 71, 73], "dot": [10, 14, 18, 19, 69], "product": [10, 11, 12, 13, 18, 19, 71], "incorrect": [10, 14, 40, 70, 71], "arxiv": [10, 14, 15, 18, 24, 32, 35], "org": [10, 14, 15, 18, 24, 32, 35, 71, 73], "ab": [10, 11, 14, 24, 36, 70, 71], "2211": [10, 35], "00593": [10, 35], "john": [10, 70, 71], "mari": [10, 70, 71], "went": [10, 70, 71], "shop": [10, 70, 71], "gave": [10, 35, 70, 71], "bag": [10, 70], "choos": [10, 70, 71], "final_ln": 10, "residual_stack_item": 10, "dure": [10, 24, 37, 39, 71, 73], "stack_activ": 10, "activation_nam": [10, 40, 70], "sublayer_typ": 10, "flexibl": 10, "given": [10, 11, 12, 13, 14, 16, 36, 37, 38, 40, 68, 69, 70, 71], "get_act_nam": [10, 69, 70, 71], "infer": [10, 14, 26, 40, 70, 71], "incl_remaind": 10, "stack_head_result": [10, 70], "axi": [10, 18, 40, 69, 70, 71], "n_head": [10, 12, 13, 14, 15, 18, 24, 38, 39, 40, 69, 70, 71, 72], "notat": [10, 70], "l0h0": 10, "stack_neuron_result": 10, "l0n0": 10, "super": [10, 14, 69, 71], "short": [10, 69, 70, 71, 73], "mostli": [10, 70, 71], "finish": [10, 14, 69, 70, 71], "oper": [10, 70, 71], "slower": 10, "unless": [10, 14, 15, 35, 71], "deprec": 10, "toggle_autodiff": 10, "toggl": [10, 14], "autodiff": [10, 71], "set_grad_en": [10, 70, 71], "danger": 10, "off": [10, 14, 35, 69, 70, 71], "realis": [10, 30, 70], "downstream": 10, "delet": [10, 69, 70], "stick": [10, 70], "mess": [10, 14, 69, 71], "inference_mod": 10, "decor": 10, "similar": [10, 12, 13, 14, 20, 24, 36, 70, 71], "requires_grad": 10, "eigenvalu": 11, "ldim": [11, 71], "mdim": [11, 71], "rdim": [11, 71], "leading_dim": [11, 69], "ba": 11, "vh": [11, 14], "collapse_l": 11, "collaps": [11, 70, 71], "left": [11, 14, 18, 69, 70, 71, 73], "orthogon": [11, 14], "self": [11, 12, 13, 14, 18, 69, 71], "collapse_r": 11, "analog": [11, 70, 71], "apart": [11, 69, 70, 71], "zero": [11, 14, 18, 36, 69, 70, 71], "bav": 11, "kv": 11, "abav": 11, "kav": 11, "av": 11, "eigenvector": [11, 71], "get_corn": [11, 69, 70], "make_even": 11, "sqrt": [11, 14, 15, 69], "diag": 11, "equival": [11, 14, 18, 70, 71], "factoris": [11, 14, 18, 71], "row": [11, 14, 40], "col": 11, "ndim": 11, "frobeniu": [11, 71], "squar": [11, 28, 29, 69, 71], "m": [11, 18, 66, 69, 70, 71], "st": 11, "transpos": [11, 69], "obviou": [11, 14, 70], "unsqueez": [11, 69], "hook": [12, 13, 14, 15, 19, 20, 21, 22, 23, 25, 26, 27, 28, 29, 32, 33, 34, 37, 70], "encod": [12, 13, 18, 31, 69, 71], "contain": [12, 13, 14, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 32, 33, 34, 35, 36, 37, 38, 39, 69, 70, 71], "bert": [12, 20, 21, 22, 32, 38, 42, 71, 72], "move_to_devic": [12, 13, 14], "kwarg": [12, 13, 14, 38, 69, 70, 71], "hookedrootmodul": [12, 13, 14, 37, 71], "hookpoint": [12, 13, 14, 37, 70, 71], "inherit": [12, 13, 71], "mvp": 12, "mask": [12, 13, 14, 18, 22, 27, 31, 33, 69, 70], "mlm": [12, 22], "next": [12, 14, 69, 70, 71], "sentenc": [12, 13, 14, 22, 32, 36, 70, 71], "nsp": 12, "dropout": [12, 13], "inconsist": [12, 13, 16], "fine": [12, 13, 71], "fold": [12, 13, 14, 18, 26, 38, 70], "ov": [12, 13, 14, 16, 18, 70, 71], "o": [12, 13, 66, 71], "qk": [12, 13, 14, 18, 70], "w_e": [12, 13, 14, 71], "conveni": [12, 13, 14, 15, 37, 69, 71], "w_e_po": [12, 14], "n_ctx": [12, 14, 15, 18, 38, 71, 72], "concaten": [12, 14, 69, 70, 71], "w_po": [12, 13, 14, 71], "overcomplet": [12, 14], "w_k": [12, 13, 14, 15, 18, 24, 71], "w_o": [12, 13, 14, 18, 19, 70, 71], "w_q": [12, 13, 14, 18, 24, 71], "w_v": [12, 13, 14, 18, 24, 71], "w_in": [12, 13, 14, 16, 71], "w_out": [12, 13, 14, 16, 71], "all_head_label": [12, 13, 14], "format": [12, 13, 14, 37, 70, 71], "h": [12, 13, 14, 70, 71], "b_k": [12, 13, 14, 18, 24, 71], "b_q": [12, 13, 14, 71], "b_u": [12, 13, 14, 70, 71], "bia": [12, 13, 14, 15, 18, 28, 29, 30, 66, 70, 71], "b_v": [12, 13, 14, 18, 24, 71], "b_in": [12, 13, 14, 71], "b_out": [12, 13, 14, 71], "buffer": [12, 13, 14], "modifi": [12, 13, 14], "cuda": [12, 13, 14, 15, 35, 38], "associ": [12, 13, 14, 37], "optim": [12, 13, 14, 63, 70], "return_typ": [12, 13, 14, 37, 70, 71], "token_type_id": [12, 21, 32], "one_zero_attention_mask": [12, 13], "binari": [12, 13, 32], "id": [12, 14, 15, 32], "belong": [12, 32], "cl": [12, 32, 71], "sep": [12, 32], "sequence_length": [12, 32, 36, 69], "attend": [12, 13, 15, 18, 19, 24, 69, 70, 71], "ignor": [12, 13, 14, 15, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 69, 71], "primarili": [12, 13], "pad": [12, 13, 14, 18, 27, 31, 33, 69, 70, 71], "shorter": [12, 13, 14, 71], "classmethod": [12, 13, 14, 15, 39, 69], "model_nam": [12, 13, 14, 15, 38, 71], "checkpoint_index": [12, 13, 14, 15, 38, 71], "checkpoint_valu": [12, 13, 14, 15, 38, 71], "hf_model": [12, 13, 14], "float32": [12, 13, 14, 15, 18, 38, 70], "from_pretrained_kwarg": [12, 13, 14], "bertformaskedlm": [12, 13], "unlik": [12, 13, 14, 40, 71], "mp": [12, 13, 14], "model_arg": [12, 13, 14, 37], "return_cache_object": [12, 13, 14], "otherwis": [12, 13, 14, 35, 36, 69], "device_or_dtyp": [12, 13, 14, 68], "print_detail": [12, 13, 14, 68, 69], "cast": [12, 13, 14], "non_block": [12, 13, 14], "memory_format": [12, 13, 14], "channels_last": [12, 13, 14], "Its": [12, 13, 14], "complex": [12, 13, 14, 15, 70, 71], "integr": [12, 13, 14, 38], "tri": [12, 13, 14, 70, 71, 73], "asynchron": [12, 13, 14], "respect": [12, 13, 14, 37, 69, 71], "host": [12, 13, 14, 38], "pin": [12, 13, 14], "desir": [12, 13, 14], "4d": [12, 13, 14], "keyword": [12, 13, 14, 37, 71], "argument": [12, 13, 14, 15, 37, 38, 69, 71], "xdoctest": [12, 13, 14], "ignore_w": [12, 13, 14], "non": [12, 13, 14, 15, 18, 35, 69, 70, 71], "determinist": [12, 13, 14, 69, 70], "nn": [12, 13, 14, 37, 71], "1913": [12, 13, 14], "3420": [12, 13, 14], "5113": [12, 13, 14], "2325": [12, 13, 14], "doubl": [12, 13, 14], "in_featur": [12, 13, 14], "out_featur": [12, 13, 14], "float64": [12, 13, 14], "env": [12, 13, 14], "torch_doctest_cuda1": [12, 13, 14], "gpu1": [12, 13, 14], "1914": [12, 13, 14], "5112": [12, 13, 14], "2324": [12, 13, 14], "float16": [12, 13, 14], "cdoubl": [12, 13, 14], "3741": [12, 13, 14], "j": [12, 13, 14, 15, 18, 38, 70, 71, 72], "2382": [12, 13, 14], "5593": [12, 13, 14], "4443": [12, 13, 14], "complex128": [12, 13, 14], "6122": [12, 13, 14], "1150": [12, 13, 14], "encoderdecod": [13, 15], "t5": [13, 15, 30, 31, 38, 42, 72], "decoder_input": 13, "decoder_po": 13, "usual": [13, 31], "fairli": [14, 70, 71], "extract": [14, 71], "harder": [14, 40, 70], "aim": [14, 70, 73], "simplifi": [14, 70, 71], "attach": [14, 71], "within": [14, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 36, 37, 40, 69, 70, 71], "inspect": [14, 70], "alter": 14, "facilit": 14, "deeper": 14, "pretrainedtokenizerbas": 14, "default_padding_sid": 14, "50": [14, 63, 71], "initialis": [14, 15], "although": [14, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37], "instanti": [14, 15, 71], "__init__": [14, 18, 19, 24, 25, 26, 28, 29, 37, 69, 71], "test_prompt": [14, 69, 70, 71], "w_gate": 14, "tokenizer_nam": [14, 15], "explicitli": [14, 15, 18, 35, 38, 40, 71], "n_devic": [14, 15, 38, 68], "greater": [14, 36], "multipl": [14, 36, 68, 69, 71], "accumulated_bia": 14, "include_mlp_bias": 14, "all_composition_scor": [14, 69], "score": [14, 15, 18, 19, 24, 36, 40, 70], "l1": 14, "h1": 14, "l2": 14, "h2": [14, 70], "upper": [14, 18], "triangular": [14, 36, 69, 71], "third": [14, 71], "pub": [14, 69], "2021": 14, "framework": [14, 18, 70, 71], "html": [14, 69, 70], "20abov": 14, "20diagram": 14, "20show": 14, "20q": 14, "2d": [14, 66, 69], "2c": 14, "20k": [14, 69], "20and": 14, "20v": 14, "2dcomposit": 14, "metric": [14, 36, 40, 70, 71], "center_unemb": [14, 70], "state_dict": 14, "center": [14, 15, 26, 28, 29, 70, 71], "subtract": [14, 36, 70], "translat": [14, 70, 71], "invari": 14, "prob": [14, 69, 70, 71], "slightli": [14, 69, 70], "misl": 14, "center_writing_weight": [14, 70, 71], "fold_layer_norm": [14, 38], "check_hooks_to_add": [14, 37], "hook_point_nam": [14, 37], "dir": [14, 37], "fwd": [14, 37], "is_perman": [14, 37], "prepend": [14, 15, 35, 37, 38, 69, 71], "overrid": [14, 15, 37, 38, 69], "fold_bias": 14, "center_weight": 14, "rm": [14, 15, 28, 29], "neighbour": 14, "further_com": [14, 15], "md": [14, 15], "fold_value_bias": 14, "alwai": [14, 15, 40, 70, 71], "constant": [14, 15, 18, 70, 71], "doesn": [14, 35, 69, 70, 71], "formal": 14, "b_o_new": 14, "b_o_origin": 14, "sum_head": 14, "b_v_head": 14, "w_o_head": 14, "loss_per_token": 14, "prepend_bo": [14, 15, 35, 38, 69, 70], "use_default_valu": 14, "padding_sid": [14, 69, 70], "start_at_lay": 14, "shortformer_pos_emb": [14, 18, 33], "attention_mask": [14, 18, 27, 31, 33, 39, 69], "stop_at_lay": 14, "past_kv_cach": [14, 27], "hookedtransformerkeyvaluecach": [14, 31, 33, 39], "flag": [14, 15, 35, 37, 40, 69, 70, 71], "entropi": [14, 69, 70, 71], "per": [14, 40, 69, 70, 71], "averag": [14, 35, 70, 71], "scalar": [14, 18, 37, 71], "default_prepend_bo": [14, 15, 35, 38, 69, 71], "bo": [14, 15, 36, 38, 69, 70, 71], "impli": 14, "usag": [14, 70], "accordingli": [14, 15, 18, 38, 70, 71], "lose": [14, 15, 38], "empir": [14, 15, 38, 40, 71], "inclus": 14, "neg": [14, 69, 70, 71], "shortform": [14, 15, 18, 33, 38], "positional_embedding_typ": [14, 15, 18], "stop": 14, "exclus": [14, 69], "etc": [14, 15, 40, 70, 71, 73], "frozen": [14, 39], "pai": [14, 18, 70], "okai": 14, "twice": [14, 35, 70, 71], "accident": [14, 37], "fold_ln": [14, 38, 70, 71], "refactor_factored_attn_matric": [14, 70], "automodelforcausallm": 14, "autoregress": [14, 63], "neo": [14, 18, 19, 24, 38, 42, 71, 72], "gptj": [14, 38, 42], "opt": [14, 38, 42, 71, 72], "solu": [14, 15, 38, 69, 71, 72], "checkpoint": [14, 15, 38, 63], "neelnanda": [14, 38], "stanford": [14, 15, 18, 19, 24, 38, 71, 72], "crfm": [14, 38, 71], "load_and_process_state_dict": 14, "alia": [14, 37, 38, 69, 71], "subsequ": [14, 38, 70, 71], "regular": [14, 18, 24], "batchnorm": [14, 70, 71], "mathemat": [14, 18, 70, 71], "w_": 14, "b_": 14, "w": [14, 67], "layernormpr": [14, 26, 29], "eff": 14, "ext": 14, "wise": [14, 36], "computation": [14, 71], "wish": 14, "defin": [14, 18, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 69, 70, 71], "x_1": [14, 71], "x_0": [14, 71], "x_2": [14, 71], "frac": [14, 71], "x_3": 14, "cdot": 14, "x_4": 14, "preced": [14, 69, 70, 71], "never": [14, 71], "w_write": 14, "keepdim": 14, "fed": [14, 36], "1000": [14, 35, 69, 71], "recreat": 14, "onto": [14, 38, 70], "By": [14, 35, 37, 38, 40, 69, 70, 71], "mix": [14, 69, 70, 71], "linearli": 14, "technic": [14, 70, 71], "deriv": [14, 71], "broadcast_b_v": 14, "broadcast": 14, "And": [14, 40, 70, 71], "destination_posit": [14, 71], "source_posit": [14, 71], "source_": 14, "destin": [14, 15, 40, 71], "behavior": [14, 15, 38, 70], "cache_dir": [14, 69], "torch_dtyp": 14, "bfloat16": 14, "boolean": [14, 37, 40, 69, 70, 71], "max_new_token": [14, 71], "stop_at_eo": 14, "eos_token_id": [14, 69], "do_sampl": 14, "top_k": [14, 69, 70, 71], "top_p": [14, 69], "temperatur": [14, 69, 71], "freq_penalti": [14, 69], "use_past_kv_cach": 14, "verbos": 14, "pos_plus_new_token": 14, "eos_token": 14, "reach": [14, 71], "avoid": [14, 15, 39, 69, 70, 71], "fiddl": 14, "rag": 14, "eot": 14, "throw": 14, "enter": [14, 70, 71, 73], "messi": [14, 71], "maximum": [14, 15, 18, 63, 71], "stable_lm": 14, "distribut": [14, 68, 69, 70, 71], "greedi": [14, 69], "search": [14, 36, 70, 71], "max": [14, 70], "mass": 14, "random": [14, 15, 35, 63, 70, 71], "temp": [14, 69], "inf": 14, "uniform": [14, 69], "frequenc": [14, 69, 70], "penalti": [14, 69], "penalis": 14, "speed": [14, 70], "applic": [14, 15, 69], "whatev": [14, 70], "tqdm": [14, 71], "get_token_posit": [14, 70, 71], "single_token": [14, 71], "present": 14, "gotcha": [14, 16, 70], "Be": 14, "care": [14, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 70, 71], "weird": [14, 15, 70, 71], "carefulli": [14, 70], "correspond": [14, 24, 36, 40, 69, 70, 71], "dummi": [14, 37, 71], "init_weight": [14, 15], "empti": [14, 37], "bulk": 14, "seed": [14, 15, 63, 71], "determin": [14, 18, 40, 68, 69, 70, 71], "NOT": [14, 37, 69, 71], "scheme": 14, "tell": [14, 35, 70, 71], "round": [14, 35, 70, 71], "18182": 14, "fan_in": [14, 69], "tha": 14, "kaim": [14, 69], "despit": [14, 71], "xavier": [14, 69], "fan_out": 14, "transformerencod": 14, "exact": 14, "72253": 14, "mup": [14, 15], "haven": 14, "2203": 14, "03466": 14, "input_to_emb": 14, "special": [14, 71], "redwood": [14, 70, 71], "load_sample_training_dataset": 14, "dataset": [14, 35, 63, 69, 71], "10k": [14, 35, 69], "get_dataset": [14, 69], "appropri": [14, 71], "info": [14, 15, 40, 69, 71], "download": [14, 69, 71], "locat": [14, 40, 70], "pt": 14, "openwebtext": [14, 35, 69], "karma": [14, 35], "reddit": [14, 35], "pile": [14, 35, 38, 69, 71, 72], "imperfectli": 14, "suppli": 14, "valid": [14, 35, 70], "loss_fn": [14, 71], "per_token": [14, 69, 71], "lm_cross_entropy_loss": [14, 69], "move_model_modules_to_devic": 14, "process_weights_": 14, "cleaner": 14, "experiment": 14, "argu": [14, 71], "somewhat": [14, 70, 71], "w_qk": [14, 18, 71], "w_ov": [14, 18, 71], "hopefulli": [14, 73], "column": [14, 40, 69], "rotat": [14, 15, 18, 71], "nth": 14, "formula": 14, "r": 14, "refactor": 14, "diagon": [14, 70, 71], "asymmetri": 14, "fiddli": 14, "preserv": [14, 70, 71], "too": [14, 40, 70], "bilinear": [14, 71], "dimension": [14, 15], "coordin": 14, "sample_datapoint": 14, "implicitli": [14, 40, 71], "hasn": 14, "manual": [14, 69, 71], "replac": [14, 15, 40, 70, 71, 73], "choic": [14, 70], "truncat": [14, 35, 69, 71], "set_token": [14, 15], "pretrainedtoken": 14, "set_use_attn_in": 14, "use_attn_in": [14, 15], "set_use_attn_result": 14, "easili": [14, 69, 70, 71], "burn": 14, "set_use_hook_mlp_in": 14, "use_hook_mlp_in": [14, 15], "set_use_split_qkv_input": 14, "use_split_qkv_input": [14, 15], "to_single_str_token": 14, "int_token": 14, "uncertain": 14, "to_token": [14, 69, 70, 71], "to_str_token": [14, 16, 70, 71], "weirdli": [14, 70, 71], "gotcha2": 14, "letter": [14, 71], "capit": [14, 70, 71], "shoot": [14, 71], "gotcha3": 14, "exce": 14, "str_token": [14, 70], "to_str": [14, 70, 71], "numpi": [14, 15, 69, 70], "arrai": [14, 16, 69], "long": [14, 71], "window": [14, 15, 69], "tokens_to_residual_direct": [14, 70], "mislead": [14, 70], "integ": [14, 69, 70, 71], "residual_direct": 14, "namedtupl": 14, "dataclass": [15, 37], "act_fn": [15, 72], "ep": 15, "1e": [15, 38], "05": [15, 38], "use_attn_scal": 15, "attn_scal": 15, "use_local_attn": 15, "original_architectur": 15, "from_checkpoint": 15, "checkpoint_label_typ": [15, 71], "window_s": [15, 18, 19, 24], "attn_typ": [15, 18, 19, 24, 30], "init_mod": 15, "normalization_typ": 15, "attention_dir": 15, "attn_onli": [15, 72], "initializer_rang": 15, "scale_attn_by_inverse_layer_idx": 15, "final_rm": 15, "d_vocab_out": [15, 34], "parallel_attn_mlp": 15, "rotary_dim": [15, 18], "n_param": [15, 72], "use_hook_token": 15, "gated_mlp": 15, "tokenizer_prepends_bo": 15, "n_key_value_head": [15, 24, 72], "post_embedding_ln": 15, "rotary_bas": 15, "10000": [15, 18, 71], "trust_remote_cod": 15, "rotary_adjacent_pair": 15, "load_in_4bit": 15, "num_expert": 15, "experts_per_token": 15, "relative_attention_max_dist": 15, "relative_attention_num_bucket": 15, "decoder_start_token_id": 15, "tie_word_embed": 15, "use_normalization_before_and_aft": 15, "attn_scores_soft_cap": 15, "output_logits_soft_cap": 15, "AND": 15, "feedforward": 15, "network": [15, 70, 71], "vocab": 15, "lowercas": 15, "relu": [15, 69, 72], "gelu": [15, 38, 71, 72], "silu": [15, 72], "gelu_new": [15, 69], "gelu_fast": [15, 69], "epsilon": 15, "5": [15, 18, 35, 36, 38, 40, 69, 70, 71, 72], "THEN": 15, "intens": 15, "divid": [15, 36, 69, 70], "distanc": [15, 18, 70], "weight_init_mod": 15, "xavier_uniform": 15, "xavier_norm": 15, "kaiming_uniform": 15, "kaiming_norm": 15, "rmspre": 15, "pipelin": 15, "parallel": [15, 69, 70], "aka": 15, "unidirect": 15, "bidirect": [15, 71], "8": [15, 18, 35, 36, 70, 71, 72], "gain": [15, 69], "layer_id": [15, 18, 19, 24, 30], "numer": [15, 16, 18, 19, 24, 71], "stabil": [15, 18, 19, 24, 71], "fp16": 15, "rotari": [15, 18], "blog": [15, 18], "eleuth": [15, 18, 69, 71], "res_stream": 15, "sinusoid": 15, "dumb": 15, "mainli": 15, "curs": 15, "init": 15, "law": 15, "pdf": [15, 18, 32, 35], "2001": 15, "08361": 15, "Will": [15, 40], "interven": [15, 37, 40, 70], "add_bos_token": [15, 69], "bitsandbyt": 15, "moe": 15, "bucket": 15, "tie": 15, "word": [15, 69, 70, 71], "gemma": [15, 38, 42, 72], "softcap": 15, "soft_cap": 15, "tanh": 15, "squash": 15, "interv": [15, 36], "from_dict": 15, "config_dict": 15, "is_layer_norm_activ": 15, "set_seed_everywher": 15, "to_dict": 15, "unwrap": [15, 69], "duplic": [15, 36, 70, 71], "get_singular_vector": 16, "vector_typ": 16, "layer_index": [16, 70], "num_vector": 16, "plot": [16, 71], "pysvelt": [16, 71], "instabl": 16, "d": [16, 35, 36, 38, 70, 72], "medium": [16, 38, 72], "svd_interpret": 16, "22": [16, 35, 69, 70, 71], "all_token": 16, "np": [16, 69, 70], "def": [16, 70, 71], "plot_matrix": 16, "filter": [16, 37, 38, 69, 71], "topk": [16, 70], "topktabl": 16, "obj_typ": 16, "abstract_attent": 17, "bert_block": 17, "bert_emb": 17, "bert_mlm_head": 17, "grouped_query_attent": 17, "layer_norm": [17, 70], "layer_norm_pr": 17, "rms_norm": 17, "rms_norm_pr": 17, "t5_attent": 17, "t5_block": 17, "token_typed_emb": 17, "transformer_block": 17, "abstractattent": [18, 19, 24, 30], "abc": [18, 71], "pure": 18, "glossari": 18, "sorri": 18, "underli": [18, 40, 70, 71], "destination_residu": 18, "destination_po": 18, "source_po": [18, 71], "abstract": [18, 70, 71], "groupedqueryattent": [18, 24], "enforc": 18, "child": 18, "better_abc": 18, "abstract_attribut": 18, "stackoverflow": 18, "question": [18, 70, 71], "23831510": 18, "256": [18, 19, 24, 71, 72], "alibi": 18, "apply_causal_mask": 18, "pos_plus_past_kv_pos_offset": 18, "past_kv_pos_offset": [18, 27, 69], "offset_po": [18, 27, 33, 69], "apply_rotari": 18, "calculate_attention_scor": [18, 24], "calculate_qkv_matric": [18, 24], "query_input": [18, 24], "key_input": [18, 24], "kv_po": [18, 30, 31], "value_input": [18, 24], "calculate_sin_cos_rotari": 18, "sine": 18, "cosin": 18, "wave": 18, "inexplic": 18, "adjac": [18, 70], "neox": [18, 38, 42, 71, 72], "clue": [18, 70], "resolv": 18, "calculate_z_scor": [18, 24], "static": [18, 35], "create_alibi_bia": 18, "head_idx": 18, "2108": 18, "12409": 18, "broad": [18, 70], "proport": [18, 69], "distant": 18, "0000": [18, 70], "0625": 18, "1250": 18, "1875": 18, "0039": 18, "0078": 18, "0117": 18, "create_alibi_multipli": 18, "geometr": 18, "ratio": [18, 69, 70, 71], "16": [18, 69, 70, 71, 72], "5000": 18, "2500": [18, 70], "0312": 18, "0156": 18, "7071": 18, "3536": 18, "1768": 18, "0884": 18, "0442": 18, "0221": 18, "0110": 18, "0055": 18, "create_alibi_slop": 18, "slope": 18, "triangl": 18, "lower": [18, 35, 36, 69, 70, 71], "bottom": [18, 71], "corner": 18, "kv_head_index": [18, 24], "past_kv_cache_entri": [18, 31, 33], "hookedtransformerkeyvaluecacheentri": [18, 31, 33, 39], "additive_attention_mask": [18, 20, 31], "position_bia": [18, 30, 31], "irrelev": [18, 70, 71], "past": [18, 39, 70], "rotate_every_two": 18, "x0": 18, "x1": 18, "param": [19, 63, 69, 71], "convent": [19, 69, 70, 71], "mistal": [19, 24], "bertblock": 20, "transformerblock": [20, 33], "except": [20, 70, 71], "overridden": [20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 69], "subclass": [20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37], "recip": [20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37], "afterward": [20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37], "former": [20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37], "regist": [20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37], "latter": [20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 71], "silent": [20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37], "bertemb": 21, "input_id": 21, "bertmlmhead": 22, "purpos": [22, 35, 70, 71], "resid": 22, "2305": 24, "13245": 24, "hood": 24, "_w_k": 24, "_w_v": 24, "getter": 24, "similarli": 24, "kept": 24, "repeat_interleav": 24, "unexpand": 24, "expan": 24, "n_query_head": 24, "gpa": 24, "normalis": [26, 70], "posemb": 27, "root": [28, 29, 71], "rmsnormpr": 29, "t5attent": [30, 31], "has_relative_attention_bia": 30, "expect": [30, 35, 36, 70, 71], "precomput": 30, "feed": [30, 35, 69, 71], "positional_bia": 30, "compute_relative_attention_bia": 30, "query_len": 30, "key_len": 30, "query_length": 30, "key_length": 30, "bin": 30, "t5block": 31, "block_index": [31, 33], "is_decod": 31, "t5layernorm": 31, "inst": 31, "encoder_additive_attention_mask": 31, "encoder_po": 31, "encoder_hidden_st": 31, "_description_": 31, "_type_": [31, 37], "tokentypeemb": 32, "1810": 32, "04805": 32, "apply_mlp": 33, "normalized_resid": 33, "central": [33, 70], "positional_embeddings_typ": 33, "canbeusedasmlp": 33, "evalu": [35, 37, 70, 71], "rough": [35, 71], "cheapli": 35, "roughli": [35, 70, 71], "baselin": 35, "ioidataset": 35, "noun": 35, "num_sampl": 35, "symmetr": 35, "ioi_ev": 35, "476": 35, "met": 35, "alic": 35, "bob": 35, "charli": 35, "ball": [35, 70], "book": 35, "397": 35, "get_default_nam": 35, "get_default_noun": 35, "get_default_templ": 35, "get_sampl": 35, "evaluate_on_dataset": 35, "data_load": 35, "induction_loss": [35, 71], "subseq_len": 35, "384": [35, 71], "io": [35, 36, 70, 71], "accuraci": [35, 36, 69], "make_code_data_load": 35, "codeparrot": [35, 69], "dump": 35, "presum": [35, 70], "natur": [35, 70, 71], "make_owt_data_load": 35, "corpu": [35, 69], "make_pile_data_load": 35, "eleutherai": [35, 38], "english": [35, 71, 73], "academ": 35, "internet": [35, 71], "make_wiki_data_load": 35, "wikitext": 35, "wikipedia": [35, 69, 71], "articl": [35, 69, 70, 71], "bother": 35, "quarantin": 35, "nowadai": 35, "leakag": 35, "though": [35, 69, 70, 71], "sanity_check": 35, "paragraph": [35, 71], "zoom": [35, 40, 70], "quick": [35, 36, 71], "saniti": [35, 70], "ok": [35, 70, 71], "gone": [35, 70, 71], "wrong": [35, 37, 70], "compute_head_attention_similarity_scor": 36, "attention_pattern": [36, 71], "detection_pattern": 36, "exclude_bo": 36, "exclude_current_token": 36, "error_measur": 36, "mul": 36, "exclude_bcurrent_token": 36, "detect_head": 36, "seq": [36, 69], "previous_token_head": 36, "duplicate_token_head": 36, "induction_head": 36, "headnam": 36, "straightforward": [36, 70], "fraction": 36, "alloc": 36, "prohibit": 36, "cours": [36, 70], "raw": [36, 70], "perfect": [36, 70], "examin": 36, "switch": 36, "advantag": 36, "closer": 36, "head_nam": 36, "ntensor": 36, "ioi": [36, 70, 71], "spacifi": 36, "analyz": 36, "paid": [36, 70, 71], "get_duplicate_token_head_detection_pattern": 36, "dynalist": 36, "n2zwtnoyhru1s4vnfsaq519j": 36, "2ukvedzonghl5uhugvhroxeo": 36, "get_induction_head_detection_pattern": 36, "_tfvup5csv5orithmqwj0gsi": 36, "get_previous_token_head_detection_pattern": 36, "0o5vohe9xezn8ertywkh7ioc": 36, "get_supported_head": 36, "hookfunct": 37, "_hookfunctionprotocol": 37, "inspir": [37, 71, 73], "garcon": [37, 71, 73], "ident": [37, 69, 70, 71], "wrap": [37, 71], "add_hook": [37, 70], "bwd": 37, "fn": 37, "hook_nam": 37, "add_perma_hook": [37, 71], "clear_context": 37, "remove_hook": 37, "including_perman": 37, "interfac": [37, 71, 73], "nice": [37, 70], "run_with_hook": [37, 70, 71], "temporari": [37, 69, 71], "debug": [37, 38, 63], "intent": 37, "reset_hook": [37, 71], "goe": [37, 70, 71], "reset_hooks_end": [37, 70], "add_caching_hook": 37, "names_filt": [37, 70], "callabl": [37, 40], "incl_bwd": 37, "namesfilt": 37, "lambda": [37, 70, 71], "cache_som": 37, "check_and_add_hook": 37, "get_caching_hook": 37, "fwd_hook": [37, 70, 71], "bwd_hook": 37, "hook_dict": 37, "exit": [37, 69], "clear": [37, 71], "reset": 37, "my_hook": 37, "hooked_loss": 37, "mod_dict": 37, "remove_all_hook_fn": 37, "model_kwarg": 37, "degrad": 37, "lenshandl": 37, "removablehandl": 37, "context_level": 37, "hold": 37, "perman": 37, "hug": 38, "face": 38, "hub": [38, 69], "768": [38, 70, 71, 72], "layer_norm_ep": 38, "init_rang": 38, "02": 38, "1024": [38, 69, 71, 72], "3072": [38, 71, 72], "12": [38, 70, 71, 72], "model_alias": 38, "01": 38, "yi": [38, 72], "34b": [38, 72], "6b": [38, 71, 72], "arthurconmi": 38, "redwood_attn_2l": [38, 72], "baidicoot": 38, "codellama": [38, 72], "hf": 38, "codellamallama": [38, 72], "3b": [38, 71, 72], "125m": [38, 71, 72], "20b": [38, 71, 72], "pythia": [38, 72], "4b": [38, 72], "dedup": [38, 72], "12b": [38, 72], "13b": [38, 71, 72], "14m": [38, 72], "160m": [38, 72], "seed1": [38, 72], "seed2": [38, 72], "seed3": [38, 72], "1b": [38, 72], "800m": 38, "8b": [38, 71, 72], "31m": [38, 72], "410m": [38, 72], "350m": 38, "9b": [38, 72], "70m": [38, 72], "19m": [38, 72], "2l512w": 38, "lr": [38, 63], "attn_only_1l512w_c4_cod": 38, "c4": [38, 69, 71], "attn_only_2l512w_c4_cod": 38, "attn_only_3l512w_c4_cod": 38, "attn_only_4l512w_c4_cod": 38, "gelu_1l512w_c4_cod": 38, "gelu_2l512w_c4_cod": 38, "gelu_3l512w_c4_cod": 38, "gelu_4l512w_c4_cod": 38, "solu_10l1280w_c4_cod": 38, "10l": [38, 71, 72], "solu_10l_v22_old": 38, "solu_12l1536w_c4_cod": 38, "12l": [38, 71, 72], "solu_12l_v23_old": 38, "solu_1l512w_c4_cod": 38, "solu_1l512w_wiki_finetun": 38, "wiki": [38, 69, 70, 71, 72], "finetun": 38, "solu_1l_v9_old": 38, "solu_2l512w_c4_cod": 38, "solu_2l_v10_old": 38, "solu_3l512w_c4_cod": 38, "solu_4l512w_c4_cod": 38, "solu_4l512w_wiki_finetun": 38, "solu_4l_v11_old": 38, "solu_6l768w_c4_cod": 38, "6l": [38, 71, 72], "solu_6l_v13_old": 38, "solu_8l1024w_c4_cod": 38, "8l": [38, 71, 72], "solu_8l_v21_old": 38, "qwen": [38, 42, 61, 72], "14b": [38, 72], "1_8b": 38, "qwen1": [38, 72], "5b": [38, 71, 72], "forev": 38, "mgpt": [38, 72], "bigcod": 38, "santacod": [38, 72], "bigscienc": 38, "1b1": [38, 72], "1b7": [38, 72], "560m": [38, 72], "7b1": [38, 72], "distilgpt2": [38, 71], "distillgpt2": [38, 72], "distil": [38, 71], "facebook": 38, "xxl": 38, "30b": [38, 71, 72], "xxxl": 38, "xl": [38, 71, 72], "66b": [38, 71, 72], "xxxxl": 38, "27b": [38, 72], "2b": [38, 72], "65b": [38, 72], "70b": [38, 72], "microsoft": 38, "phi": [38, 42, 59, 72], "mini": 38, "4k": 38, "1_5": [38, 72], "roneneldan": 38, "tinystori": 38, "1layer": 38, "21m": [38, 72], "28m": [38, 72], "2layer": 38, "33m": [38, 72], "3m": [38, 72], "8m": [38, 72], "instuct": 38, "stabilityai": 38, "stablelm": [38, 71, 72], "alpha": [38, 72], "x21": 38, "arwen": 38, "battlestar": 38, "x49": 38, "beren": 38, "caprica": 38, "x81": 38, "celebrimbor": 38, "darkmatt": 38, "x343": 38, "durin": 38, "eowyn": 38, "x777": 38, "expans": 38, "alias": 38, "non_hf_hosted_model_nam": 38, "qwen2": [38, 42, 72], "get_checkpoint_label": [38, 71], "label_typ": 38, "get_num_params_of_pretrain": 38, "suffici": [38, 70], "get_pretrained_model_config": 38, "hf_cfg": 38, "automodel": 38, "autoconfig": 38, "infrastructur": [38, 70, 71, 73], "ourselv": [39, 69, 71, 73], "previous_attention_mask": 39, "pos_so_far": 39, "append": [39, 70, 71], "prefix": 39, "append_attention_mask": 39, "new_token": 39, "freez": 39, "init_cach": 39, "unfreez": 39, "past_kei": 39, "jaxtyp": [39, 70, 71], "past_valu": 39, "new_kei": 39, "new_valu": 39, "init_cache_entri": 39, "structur": [40, 71], "generic_activation_patch": 40, "specialis": [40, 70], "introduc": [40, 70], "rome": [40, 70, 71], "baulab": 40, "corrupt": [40, 70, 71], "iter": [40, 69, 70, 71], "localis": [40, 70, 71], "__from__": 40, "__to": 40, "__the": 40, "confid": [40, 70, 71], "intuit": [40, 70, 71], "diffus": [40, 70], "spread": [40, 70], "connect": [40, 70], "ultim": [40, 70], "tend": [40, 71], "extrem": [40, 70, 71, 73], "eiffel": 40, "tower": 40, "pari": 40, "factual": [40, 70], "recal": [40, 70], "colosseum": 40, "corrupted_token": [40, 70, 71], "clean_cach": [40, 70, 71], "patching_metr": 40, "patch_sett": 40, "index_axis_nam": 40, "src_po": [40, 70], "dest_po": [40, 70, 71], "index_df": 40, "datafram": 40, "return_index_df": 40, "counterfactu": [40, 70, 71], "Then": 40, "index_to_act_nam": 40, "recov": [40, 70, 71], "diff": [40, 70], "corrupted_activ": 40, "chunk": 40, "fill": 40, "flatten": [40, 70, 71], "patched_output": 40, "get_act_patch_attn_head_all_pos_everi": 40, "patch_typ": 40, "get_act_patch_attn_head_by_pos_everi": 40, "get_act_patch_attn_head_k_all_po": 40, "corruptedactiv": 40, "patchedactiv": 40, "layer_head_vector_patch_sett": 40, "axisnam": 40, "get_act_patch_attn_head_k_by_po": 40, "layer_pos_head_vector_patch_sett": 40, "get_act_patch_attn_head_out_all_po": 40, "get_act_patch_attn_head_out_by_po": 40, "get_act_patch_attn_head_pattern_all_po": 40, "layer_head_pattern_patch_sett": 40, "get_act_patch_attn_head_pattern_by_po": 40, "layer_head_pos_pattern_patch_sett": 40, "get_act_patch_attn_head_pattern_dest_src_po": 40, "layer_head_dest_src_pos_pattern_patch_sett": 40, "get_act_patch_attn_head_q_all_po": 40, "get_act_patch_attn_head_q_by_po": 40, "get_act_patch_attn_head_v_all_po": 40, "get_act_patch_attn_head_v_by_po": 40, "get_act_patch_attn_out": 40, "layer_pos_patch_sett": 40, "get_act_patch_block_everi": 40, "get_act_patch_mlp_out": 40, "get_act_patch_resid_mid": 40, "get_act_patch_resid_pr": 40, "clean_activ": 40, "weight_convers": 41, "coder": 42, "mingpt": 42, "nanogpt": 42, "neel_solu_old": 42, "phi3": 42, "convert_bloom_weight": 44, "convert_coder_weight": 45, "convert_mistral_weight": 51, "convert_mixtral_weight": 52, "convert_phi_weight": 58, "convert_phi3_weight": 59, "convert_qwen_weight": 60, "convert_qwen2_weight": 61, "convert_t5_weight": 62, "hookedtransformertrainconfig": 63, "num_epoch": 63, "001": 63, "max_grad_norm": 63, "weight_decai": 63, "optimizer_nam": 63, "adam": 63, "warmup_step": 63, "save_everi": 63, "save_dir": 63, "wandb": 63, "wandb_project_nam": 63, "print_everi": 63, "max_step": 63, "hyperparamet": [63, 69], "epoch": 63, "rate": [63, 71], "decai": 63, "warmup": 63, "wandb_project": 63, "termin": 63, "activation_funct": 64, "addmm": 64, "batch_addmm": 66, "d_out": [66, 69], "d_in": [66, 69], "fuse": 66, "conv1d": 66, "9ba9369a2557e53a01378199a9839ec6e82d8bc7": 66, "src": 66, "pytorch_util": 66, "l102": 66, "l106": 66, "vanilla_addmm": 66, "mat1": 66, "mat2": 66, "typecheck": 66, "complex_attn_linear": 67, "almost": [67, 70], "simple_attn_linear": 67, "extra": [67, 69], "assist": 68, "get_device_for_block_index": 68, "target": 68, "move_to_and_update_config": 68, "vari": [69, 70], "throughout": [69, 71], "locallyoverridendefault": 69, "restor": 69, "overriden": 69, "input_slic": 69, "syntax": [69, 70, 71], "reduc": [69, 70, 71], "leav": [69, 71], "elif": 69, "1d": 69, "sliceinput": 69, "valueerror": 69, "abov": [69, 70, 71], "max_ctx": 69, "int32": 69, "slice_input": 69, "calc_fan_in_and_fan_out": 69, "fan": 69, "composition_scor": 69, "broadcast_dim": 69, "leading_dims_left_and_right": 69, "download_file_from_hf": 69, "repo_nam": 69, "file_nam": 69, "subfold": 69, "home": 69, "runner": 69, "force_is_torch": 69, "json": 69, "pth": 69, "extens": [69, 70], "layer_typ": [69, 70], "shorthand": 69, "loop": [69, 70, 71, 73], "hack": [69, 71], "stuff": [69, 71], "readabl": 69, "digit": [69, 71], "k6": 69, "scale4ln1": 69, "appear": [69, 71], "distinguish": [69, 70], "hook_k": [69, 71], "hook_pr": [69, 71], "27": [69, 70, 71], "hook_norm": [69, 71], "pre5": 69, "get_attention_mask": 69, "leftmost": 69, "rightmost": 69, "consid": 69, "get_cumsum_along_dim": 69, "dataset_nam": 69, "000": [69, 71], "enorm": [69, 71], "100gb": 69, "2tb": 69, "effort": [69, 70], "dataload": 69, "fanci": 69, "data_dir": 69, "approx": [69, 70, 71], "ton": [69, 73], "divers": [69, 70, 71], "coloss": 69, "crawl": 69, "bigger": 69, "c4_code": 69, "friendli": 69, "22m": [69, 71], "5m": 69, "20220301": 69, "en": [69, 71], "get_devic": [69, 70, 71], "get_input_with_manually_prepended_bo": 69, "autotoken": 69, "get_nested_attr": 69, "obj": 69, "attr_str": 69, "retriev": 69, "nest": 69, "hierarchi": 69, "get_offset_position_id": 69, "offset": [69, 70, 71], "get_tokenizer_with_bo": 69, "Such": [69, 70], "llamatoken": 69, "get_tokens_with_bos_remov": 69, "init_kaiming_normal_": 69, "nonlinear": 69, "std": 69, "init_kaiming_uniform_": 69, "init_xavier_normal_": 69, "init_xavier_uniform_": 69, "is_lower_triangular": 69, "is_squar": 69, "keep_single_column": 69, "col_nam": 69, "lm_accuraci": 69, "seq_len": [69, 70, 71], "altern": 69, "override_or_use_default_valu": 69, "default_flag": 69, "print_gpu_mem": 69, "step_nam": 69, "repeat_along_head_dimens": 69, "clone_tensor": 69, "sample_logit": 69, "final_logit": [69, 70], "vocab_s": 69, "high": [69, 70, 71], "argmaxi": 69, "90": 69, "renormalis": 69, "mutual": 69, "neither": [69, 70], "input_token": 69, "todo": 69, "edg": 69, "randn": [69, 71], "uniqu": 69, "return_count": 69, "set_nested_attr": 69, "prepend_space_to_answ": 69, "eleph": 69, "endoftext": [69, 70, 71], "14": [69, 70, 71, 72], "51": [69, 71], "0th": [69, 70], "59": [69, 71, 72], "ground": [69, 70], "1th": [69, 70], "41": [69, 71], "tree": 69, "2th": [69, 70], "3th": [69, 70], "45": [69, 71], "car": 69, "4th": [69, 70], "13": [69, 70, 71], "92": [69, 70], "55": [69, 70, 71], "river": 69, "5th": [69, 70], "79": 69, "25": [69, 70, 71, 72], "street": 69, "6th": [69, 70], "77": 69, "21": [69, 70, 71], "7th": [69, 70], "75": 69, "hill": 69, "8th": [69, 70], "swing": 69, "9th": [69, 70], "46": [69, 71, 72], "61": [69, 72], "park": [69, 70], "to_numpi": [69, 70, 71], "tokenize_and_concaten": 69, "max_length": 69, "column_nam": 69, "num_proc": 69, "eo": [69, 71], "reshap": [69, 70], "____": 69, "drop": [69, 71], "faster": [69, 70, 71], "parallelis": [69, 71], "chop": 69, "privileg": 69, "earli": [69, 71], "cnn": [69, 71], "bos_token_id": 69, "swap": [69, 70], "runtim": [70, 71], "hardwar": [70, 71], "pane": [70, 71], "sidebar": [70, 71], "navig": [70, 71], "vscode": [70, 71], "outlin": 70, "tab": 70, "dropdown": [70, 71], "arrow": [70, 71], "page": [70, 71], "ctrl": [70, 71], "in_colab": [70, 71], "circuitsvi": [70, 71], "node": [70, 71], "curl": [70, 71], "fssl": [70, 71], "deb": [70, 71], "nodesourc": [70, 71], "setup_16": [70, 71], "sudo": [70, 71], "bash": [70, 71], "apt": [70, 71], "nodej": [70, 71], "noqa": [70, 71], "ipython": [70, 71], "get_ipython": [70, 71], "ip": [70, 71], "extension_manag": [70, 71], "autoreload": [70, 71], "functool": [70, 71], "plotli": [70, 71], "express": [70, 71], "px": [70, 71], "pio": [70, 71], "attention_head": 70, "fancy_einsum": [70, 71], "ifram": 70, "differenti": [70, 71], "simplic": 70, "imshow": [70, 71], "color_continuous_midpoint": [70, 71], "color_continuous_scal": [70, 71], "rdbu": [70, 71], "scatter": [70, 71], "xaxi": [70, 71], "yaxi": [70, 71], "caxi": [70, 71], "color": [70, 71], "principl": [70, 71, 73], "fun": [70, 71, 73], "gap": [70, 71, 73], "plai": [70, 71, 73], "flow": [70, 71, 73], "toolkit": [70, 71], "stylist": 70, "slowli": 70, "convei": 70, "tag": 70, "asid": 70, "flavour": 70, "weed": 70, "star": 70, "tagexampl": 70, "capabl": [70, 71], "interview": [70, 71], "kevin": [70, 71], "wang": 70, "twitter": 70, "thread": 70, "overview": 70, "bottl": [70, 71], "milk": [70, 71], "26": [70, 71, 72], "Their": 70, "skimp": 70, "rigour": 70, "suggest": 70, "evid": 70, "80m": [70, 71], "simplif": 70, "nbval_ignore_output": [70, 71], "stabl": 70, "example_prompt": 70, "example_answ": 70, "39": [70, 71], "lt": [70, 71], "gt": [70, 71], "09": [70, 71], "70": 70, "07": [70, 71], "38": [70, 71], "67": 70, "35": [70, 71], "54": [70, 71], "11": [70, 71, 72], "84": [70, 71], "73": 70, "hi": [70, 71], "06": 70, "her": [70, 71], "74": 70, "52": [70, 71, 72], "49": [70, 71], "jesu": 70, "97": 70, "42": [70, 71, 72], "him": 70, "subword": 70, "frequent": 70, "substr": [70, 71], "headach": 70, "annoi": [70, 71], "devot": 70, "sensibl": 70, "later": [70, 71], "wherev": 70, "flesh": 70, "prompt_format": 70, "jame": 70, "dan": 70, "sid": 70, "appl": 70, "martin": 70, "ami": 70, "drink": 70, "correct_token": 70, "insert": 70, "filler": 70, "newlin": 70, "intellig": 70, "complic": 70, "adjust": [70, 71], "aggreg": 70, "original_logit": 70, "upon": 70, "subject": [70, 71], "logits_to_ave_logit_diff": 70, "per_prompt": 70, "answer_logit": 70, "gather": 70, "answer_logit_diff": 70, "detach": [70, 71], "decim": [70, 71], "original_average_logit_diff": 70, "3370": 70, "2020": 70, "7090": 70, "7970": 70, "7200": 70, "2810": 70, "6010": 70, "7670": 70, "552": 70, "33": [70, 71], "dive": 70, "spend": [70, 71], "engag": 70, "decent": [70, 71], "hypothes": 70, "cheat": [70, 71], "hypothesi": 70, "scienc": 70, "belief": 70, "trap": 70, "flounder": 70, "dogmat": 70, "overconfid": 70, "unwil": 70, "contradict": 70, "flinch": 70, "disconfirm": 70, "focu": 70, "primit": 70, "nearbi": 70, "came": 70, "trigram": 70, "symmetri": 70, "cancel": 70, "inhibit": 70, "spoiler": 70, "simplist": 70, "importantli": [70, 71], "perfectli": [70, 71], "final_residual_stream": 70, "eleg": 70, "particularli": 70, "aspect": 70, "nicer": 70, "inde": 70, "log_prob": 70, "log_softmax": 70, "logsumexp": 70, "decid": 70, "pronoun": 70, "refin": 70, "friendlier": 70, "answer_residual_direct": 70, "logit_diff_direct": 70, "account": 70, "w_u_fold": 70, "unigram": [70, 71], "statist": [70, 71], "opposit": 70, "hook_normalis": 70, "sub_layer_typ": 70, "final_token_residual_stream": 70, "scaled_final_token_residual_stream": 70, "average_logit_diff": 70, "residual_stack_to_logit_diff": 70, "scaled_residual_stack": 70, "fascinatingli": 70, "utterli": 70, "unabl": 70, "hover": [70, 71], "n_pre": 70, "n_mid": 70, "n_post": 70, "middl": [70, 71], "accumulated_residu": 70, "logit_lens_logit_diff": 70, "arang": 70, "hover_nam": [70, 71], "terminologi": 70, "overload": 70, "kth": 70, "per_layer_residu": 70, "per_layer_logit_diff": 70, "independ": [70, 71, 73], "l9h6": 70, "l9h9": 70, "l10h7": 70, "l11h10": 70, "harm": 70, "strongli": 70, "observ": [70, 71], "144": 70, "claim": 70, "surpris": 70, "7x": 70, "per_head_residu": 70, "per_head_logit_diff": 70, "rearrang": 70, "weren": 70, "alan": [70, 71], "coonei": [70, 71], "illustr": [70, 71], "mistak": 70, "mayb": [70, 71], "sai": [70, 71], "summari": 70, "sole": 70, "visualize_attention_pattern": 70, "local_cach": 70, "local_token": 70, "max_width": 70, "700": 70, "isinst": 70, "batch_index": 70, "combin": [70, 71], "attention_head_nam": 70, "show_cod": 70, "title_html": 70, "br": 70, "div": 70, "width": [70, 71], "top_positive_logit_attr_head": 70, "positive_html": 70, "top_negative_logit_attr_head": 70, "negative_html": 70, "conceptu": 70, "clearli": 70, "compos": [70, 71], "ideal": [70, 71], "david": [70, 71], "bau": [70, 71], "meng": [70, 71], "trace": [70, 71], "anim": 70, "lai": 70, "pro": 70, "con": 70, "Or": 70, "bake": 70, "claus": 70, "tack": 70, "gaussian": 70, "nois": 70, "beforehand": 70, "19": [70, 71], "corrupted_prompt": [70, 71], "corrupted_logit": [70, 71], "corrupted_cach": 70, "corrupted_average_logit_diff": 70, "temporarili": [70, 71], "patch_residual_compon": 70, "corrupted_residual_compon": 70, "normalize_patched_logit_diff": 70, "patched_logit_diff": [70, 71], "wors": [70, 71], "patched_residual_stream_diff": 70, "hook_fn": 70, "patched_logit": [70, 71], "abus": 70, "prompt_position_label": 70, "tok": 70, "_": [70, 71], "enumer": [70, 71], "reus": 70, "patched_attn_diff": 70, "patched_mlp_diff": 70, "patched_attn_logit": 70, "patched_attn_logit_diff": 70, "patched_mlp_logit": 70, "patched_mlp_logit_diff": 70, "late": [70, 71], "contrast": 70, "statement": 70, "mlp0": 70, "destroi": 70, "frame": 70, "unprincipl": 70, "invers": [70, 71], "plausibli": 70, "dedic": 70, "overcom": 70, "love": 70, "someon": 70, "patch_head_vector": 70, "corrupted_head_vector": 70, "patched_head_z_diff": 70, "l8h6": 70, "l8h10": 70, "l7h9": 70, "l5h5": 70, "l6h9": 70, "l3h0": 70, "semi": 70, "disentangl": 70, "familiar": 70, "28": [70, 71, 72], "patched_head_v_diff": 70, "heatmap": 70, "29": [70, 71], "lesson": 70, "head_label": 70, "range_x": 70, "range_i": 70, "31": [70, 71], "patch_head_pattern": 70, "corrupted_head_pattern": 70, "patched_head_attn_diff": 70, "32": [70, 71, 72], "reconsolid": 70, "extend": 70, "l7h3": 70, "specul": 70, "mysteri": [70, 71], "top_heads_by_output_patch": 70, "first_mid_lay": 70, "first_late_lay": 70, "early_head": 70, "mid_head": 70, "logical_and": 70, "late_head": 70, "diagram": 70, "l1h2": 70, "latest": 70, "definit": 70, "priori": 70, "stroke": 70, "didn": 70, "bracket": 70, "serv": [70, 71], "particip": 70, "behav": 70, "l5h0": 70, "wrote": [70, 71, 73], "overkil": 70, "simpler": 70, "repurpos": 70, "machineri": 70, "life": [70, 71], "built": 70, "34": [70, 71], "example_text": [70, 71], "seek": 70, "machin": [70, 71], "example_repeated_text": 70, "example_repeated_token": 70, "example_repeated_logit": 70, "example_repeated_cach": 70, "induction_head_label": 70, "81": 70, "65": 70, "800": 70, "accord": 70, "wildli": 70, "characteris": 70, "superfici": 70, "boost": [70, 71], "anti": 70, "suppress": [70, 71], "pick": [70, 71], "signal": 70, "hook_": 70, "hook_attn": 70, "token_po": 70, "metadata": 70, "36": [70, 71, 72], "prev_token_scor": 70, "prev_token_hook": 70, "dim1": [70, 71], "dim2": [70, 71], "duplicate_token_scor": 70, "duplicate_token_hook": 70, "induction_scor": [70, 71], "induction_hook": 70, "manual_se": [70, 71], "original_token": 70, "randint": [70, 71], "20000": [70, 71], "repeated_token": [70, 71], "pattern_filt": 70, "act_nam": [70, 71], "endswith": [70, 71], "hook_pattern": [70, 71], "0390": 70, "0310": 70, "1890": 70, "1720": 70, "0680": 70, "1570": 70, "0210": 70, "4820": 70, "0030": 70, "1320": 70, "0050": 70, "0020": 70, "0090": 70, "0040": 70, "0010": 70, "instantli": 70, "37": [70, 71], "seen": [70, 71], "mosaic": 70, "40": [70, 71, 72], "fascin": 70, "knock": 70, "naiv": [70, 71], "convers": 70, "flaw": 70, "knockout": 70, "send": 70, "redund": 70, "job": 70, "underestim": 70, "57": [70, 71], "99": [70, 71], "hook_z": [70, 71], "top_name_mov": 70, "top_name_mover_lay": 70, "top_name_mover_head": 70, "ablate_top_head_hook": 70, "ablated_logit": 70, "ablated_cach": 70, "2f": [70, 71], "l10h10": 70, "margin": 70, "obvious": 70, "per_head_ablated_residu": 70, "per_head_ablated_logit_diff": 70, "04": [70, 71], "uniformli": [70, 71], "042": 70, "5200": 70, "4700": 70, "8200": 70, "5100": 70, "2600": 70, "1800": 70, "4300": 70, "5700": 70, "3500": 70, "2900": 70, "6800": 70, "4900": 70, "8700": 70, "4200": 70, "reader": [70, 71], "gentler": 71, "tip": 71, "development_mod": 71, "in_github": 71, "getenv": 71, "github_act": 71, "render": 71, "argh": 71, "notebook_connect": 71, "cv": 71, "hello": 71, "auto": 71, "autograd": 71, "grad_mod": 71, "0x7f208dc6d3d0": 71, "speak": [71, 73], "human": [71, 73], "palm": [71, 73], "nor": [71, 73], "offend": [71, 73], "anthrop": [71, 73], "team": [71, 73], "got": [71, 73], "frustrat": [71, 73], "deepspe": [71, 73], "industri": [71, 73], "heavili": [71, 73], "credit": [71, 73], "nelson": [71, 73], "elhag": [71, 73], "chri": [71, 73], "olah": [71, 73], "model_description_text": 71, "hyper": 71, "1758": 71, "box": 71, "On": 71, "insid": 71, "kinda": 71, "gpt2_cache_no_batch_dim": 71, "gpt2_cach": 71, "gpt2_text": 71, "summar": 71, "supervis": 71, "taskspecif": 71, "gpt2_token": 71, "gpt2_logit": 71, "lock": 71, "grid": 71, "gpt2_str_token": 71, "neural": 71, "system": 71, "surgic": 71, "surround": 71, "current_activation_valu": 71, "new_activation_valu": 71, "substitut": 71, "relationship": 71, "underr": 71, "janki": 71, "shamelessli": 71, "probepoint": 71, "qualiti": 71, "head_ablation_hook": 71, "layer_to_abl": 71, "head_index_to_abl": 71, "original_loss": 71, "ablated_loss": 71, "3f": 71, "999": 71, "453": 71, "stai": 71, "clean_prompt": 71, "clean_token": 71, "logits_to_logit_diff": 71, "correct_answ": 71, "incorrect_answ": 71, "correct_index": 71, "incorrect_index": 71, "clean_logit": 71, "clean_logit_diff": 71, "corrupted_logit_diff": 71, "276": 71, "738": 71, "residual_stream_patching_hook": 71, "clean_resid_pr": 71, "num_posit": 71, "ioi_patching_result": 71, "temp_hook_fn": 71, "ish": 71, "token_label": 71, "workflow": 71, "michael": 71, "jordan": 71, "surnam": 71, "occurr": 71, "terribl": 71, "halfwai": 71, "input_tensor": 71, "random_token": 71, "repeated_logit": 71, "correct_log_prob": 71, "loss_by_posit": 71, "manipul": 71, "hook_funct": 71, "induction_score_stor": 71, "induction_score_hook": 71, "induction_strip": 71, "pattern_hook_names_filt": 71, "highli": 71, "stripe": 71, "induction_head_lay": 71, "induction_head_index": 71, "single_random_sequ": 71, "repeated_random_sequ": 71, "visualize_pattern_hook": 71, "3d": 71, "four": 71, "300m": 71, "soon": 71, "distilgpt": 71, "distilgpt2_induction_score_stor": 71, "classic": 71, "openai": 71, "85m": [71, 72], "700m": 71, "22b": 71, "300b": 71, "180b": 71, "600": 71, "265": 71, "108m": 71, "bookscorpu": 71, "free": 71, "512": [71, 72], "tractabl": 71, "motif": 71, "80": [71, 72], "shuffl": 71, "scan": 71, "40m": 71, "100m": 71, "200m": 71, "340m": [71, 72], "15b": 71, "13m": [71, 72], "digress": 71, "usefulli": 71, "variengien": 71, "websit": 71, "cleantransformerdemo": 71, "new_activ": 71, "old_activ": 71, "remind": 71, "50267": 71, "named_paramet": 71, "startswith": 71, "fallback": 71, "spam": 71, "dest_posit": 71, "brown": 71, "fox": 71, "lazi": 71, "dog": 71, "num": 71, "print_name_shape_hook_funct": 71, "not_in_late_block_filt": 71, "hook_q": 71, "hook_v": 71, "hook_attn_scor": 71, "hook_attn_out": 71, "hook_resid_mid": 71, "hook_post": 71, "hook_mlp_out": 71, "hook_resid_post": 71, "preconcept": 71, "overhead": 71, "elementwis": 71, "consequ": 71, "rare": 71, "dramat": 71, "degre": 71, "punctuat": 71, "ass": 71, "randomredditor": 71, "unembed_bia": 71, "bias_valu": 71, "bias_indic": 71, "repr": 71, "03": 71, "98": 71, "68": 71, "48": [71, 72], "47": 71, "72": [71, 72], "44": [71, 72], "82": 71, "\u30b5\u30fc\u30c6\u30a3": 71, "83": 71, "x18": 71, "x14": 71, "\u9f8d": 71, "x1b": 71, "x05": 71, "x00": 71, "x06": 71, "x07": 71, "x0c": 71, "x02": 71, "oreandonlin": 71, "x11": 71, "x10": 71, "favour": 71, "6x": 71, "john_bia": 71, "mary_bia": 71, "4f": 71, "exp": 71, "8995": 71, "6034": 71, "6550x": 71, "finit": 71, "invert": 71, "de": 71, "uncommon": 71, "iz": 71, "charact": 71, "example_text_str_token": 71, "example_text_token": 71, "50256": 71, "464": 71, "717": 71, "1517": 71, "345": 71, "761": 71, "284": 71, "3785": 71, "503": 71, "318": 71, "1635": 71, "4919": 71, "1243": 71, "389": 71, "11241": 71, "1143": 71, "4600": 71, "19849": 71, "1462": 71, "62": 71, "2536": 71, "482": 71, "641": 71, "63": 71, "30778": 71, "257": 71, "4731": 71, "656": 71, "262": 71, "16326": 71, "292": 71, "1351": 71, "286": 71, "850": 71, "37336": 71, "25666": 71, "290": 71, "523": 71, "8781": 71, "7301": 71, "644": 71, "2420": 71, "3073": 71, "588": 71, "1675": 71, "10176": 71, "428": 71, "1309": 71, "338": 71, "779": 71, "340": 71, "319": 71, "7322": 71, "signifi": 71, "example_multi_text": 71, "cat": 71, "sat": 71, "mat": 71, "example_multi_text_token": 71, "3797": 71, "3332": 71, "2603": 71, "1107": 71, "1327": 71, "th": 71, "cat_text": 71, "cat_logit": 71, "cat_prob": 71, "capital_the_token_index": 71, "ascii": 71, "squeez": 71, "annoy": 71, "arithmet": 71, "impress": 71, "2342": 71, "2017": 71, "21445": 71, "1000000": 71, "999999": 71, "214": 71, "000000": 71, "9999": 71, "tim": 71, "ne": 71, "el": 71, "messier": 71, "takeawai": 71, "unexpect": 71, "notic": 71, "trip": 71, "confusingli": 71, "forth": 71, "ioi_logits_with_bo": 71, "clair": 71, "mary_logit_with_bo": 71, "claire_logit_with_bo": 71, "ioi_logits_without_bo": 71, "mary_logit_without_bo": 71, "claire_logit_without_bo": 71, "754": 71, "782": 71, "air": 71, "understood": 71, "requisit": 71, "attention_scor": 71, "ab_factor": 71, "9105": 71, "linalg": 71, "eig": 71, "2877e": 71, "00": 71, "8626e": 71, "3121e": 71, "9038e": 71, "08": 71, "1527e": 71, "2877": 71, "3121": 71, "3126e": 71, "3963e": 71, "2029e": 71, "7690e": 71, "2164e": 71, "3126": 71, "3963": 71, "300": 71, "abc_factor": 71, "unfactor": 71, "160": 71, "0830": 71, "43": 71, "ab_unfactor": 71, "isclos": 71, "subspac": 71, "coincid": 71, "assert": 71, "negat": 71, "proxi": 71, "lambda_i": 71, "ov_circuit_all_head": 71, "ov_circuit_all_heads_eigenvalu": 71, "complex64": 71, "ov_copying_scor": 71, "zmax": 71, "zmin": 71, "l11h11": 71, "imag": 71, "imaginari": 71, "full_ov_circuit": 71, "full_ov_circuit_eigenvalu": 71, "full_ov_copying_scor": 71, "interestingli": 71, "correl": 71, "outlier": 71, "ansh": 71, "radhakrishnan": 71, "establish": 71, "53": 71, "presid": 71, "barack": 71, "obama": 71, "caught": 71, "embarrass": 71, "scandal": 71, "nthe": 71, "financi": 71, "wife": 71, "chelsea": 71, "she": 71, "woman": 71, "lightweight": 71, "squarethenadd": 71, "hook_squar": 71, "twolayermodel": 71, "layer1": 71, "layer2": 71, "hook_in": 71, "hook_mid": 71, "hook_out": 71, "x_in": 71, "x_mid": 71, "x_out": 71, "model_out": 71, "cache_object": 71, "780": 71, "784": 71, "56": [71, 72], "set_to_zero_hook": 71, "num_checkpoint": 71, "piecewis": 71, "schedul": 71, "crash": 71, "11b": [71, 72], "centr": 71, "hoc": 71, "count": 71, "checkpoint_label": 71, "log_i": 71, "marker": 71, "brief": 71, "suddenli": 71, "500": 71, "visibl": 71, "curv": 71, "briefli": 71, "deliber": 71, "justic": 71, "chosen": 71, "60": [71, 72], "500m": 71, "58": 71, "arbitrarili": 71, "fast": 71, "checkpoint_indic": 71, "checkpointed_model": 71, "tokens_trained_on": 71, "model_for_this_checkpoint": 71, "tokens_seen_for_this_checkpoint": 71, "induction_loss_for_this_checkpoint": 71, "contextualis": 71, "strategi": 71, "95": 71, "log_x": 71, "302m": 72, "4096": 72, "708m": 72, "1280": 72, "5120": 72, "1600": 72, "6400": 72, "42m": 72, "2048": 72, "50272": 72, "8192": 72, "2560": 72, "10240": 72, "128": 72, "16384": 72, "20480": 72, "7168": 72, "28672": 72, "9216": 72, "36864": 72, "50400": 72, "6144": 72, "50432": 72, "96": 72, "24576": 72, "2m": 72, "50304": 72, "7m": 72, "805m": 72, "50688": 72, "50278": 72, "736": 72, "2944": 72, "101m": 72, "197m": 72, "1536": 72, "48262": 72, "4m": 72, "0m": 72, "50277": 72, "524k": 72, "50259": 72, "32000": 72, "11008": 72, "13824": 72, "32b": 72, "6656": 72, "17920": 72, "22016": 72, "78b": 72, "32016": 72, "128256": 72, "14336": 72, "25m": 72, "28996": 72, "393k": 72, "6m": 72, "47b": 72, "32768": 72, "250880": 72, "679m": 72, "0b": 72, "49280": 72, "151936": 72, "5504": 72, "152064": 72, "13696": 72, "308m": 72, "2816": 72, "6912": 72, "391m": 72, "896": 72, "4864": 72, "8960": 72, "3584": 72, "18944": 72, "51200": 72, "32064": 72, "256000": 72, "2304": 72, "gelu_pytorch_tanh": 72, "4608": 72, "64000": 72, "39b": 72, "32128": 72, "100000": 72, "formerli": 73, "transfer": 73}, "objects": {"transformer_lens": [[10, 0, 0, "-", "ActivationCache"], [11, 0, 0, "-", "FactoredMatrix"], [12, 0, 0, "-", "HookedEncoder"], [13, 0, 0, "-", "HookedEncoderDecoder"], [14, 0, 0, "-", "HookedTransformer"], [15, 0, 0, "-", "HookedTransformerConfig"], [16, 0, 0, "-", "SVDInterpreter"], [35, 0, 0, "-", "evals"], [36, 0, 0, "-", "head_detector"], [37, 0, 0, "-", "hook_points"], [38, 0, 0, "-", "loading_from_pretrained"], [39, 0, 0, "-", "past_key_value_caching"], [40, 0, 0, "-", "patching"], [63, 0, 0, "-", "train"], [69, 0, 0, "-", "utils"]], "transformer_lens.ActivationCache": [[10, 1, 1, "", "ActivationCache"]], "transformer_lens.ActivationCache.ActivationCache": [[10, 2, 1, "", "accumulated_resid"], [10, 2, 1, "", "apply_ln_to_stack"], [10, 2, 1, "", "apply_slice_to_batch_dim"], [10, 2, 1, "", "compute_head_results"], [10, 2, 1, "", "decompose_resid"], [10, 2, 1, "", "get_full_resid_decomposition"], [10, 2, 1, "", "get_neuron_results"], [10, 2, 1, "", "items"], [10, 2, 1, "", "keys"], [10, 2, 1, "", "logit_attrs"], [10, 2, 1, "", "remove_batch_dim"], [10, 2, 1, "", "stack_activation"], [10, 2, 1, "", "stack_head_results"], [10, 2, 1, "", "stack_neuron_results"], [10, 2, 1, "", "to"], [10, 2, 1, "", "toggle_autodiff"], [10, 2, 1, "", "values"]], "transformer_lens.FactoredMatrix": [[11, 1, 1, "", "FactoredMatrix"]], "transformer_lens.FactoredMatrix.FactoredMatrix": [[11, 3, 1, "", "AB"], [11, 3, 1, "", "BA"], [11, 3, 1, "", "S"], [11, 3, 1, "", "T"], [11, 3, 1, "", "U"], [11, 3, 1, "", "Vh"], [11, 2, 1, "", "collapse_l"], [11, 2, 1, "", "collapse_r"], [11, 3, 1, "", "eigenvalues"], [11, 2, 1, "", "get_corner"], [11, 2, 1, "", "make_even"], [11, 3, 1, "", "ndim"], [11, 2, 1, "", "norm"], [11, 3, 1, "", "pair"], [11, 2, 1, "", "svd"], [11, 2, 1, "", "unsqueeze"]], "transformer_lens.HookedEncoder": [[12, 1, 1, "", "HookedEncoder"]], "transformer_lens.HookedEncoder.HookedEncoder": [[12, 3, 1, "", "OV"], [12, 3, 1, "", "QK"], [12, 3, 1, "", "W_E"], [12, 3, 1, "", "W_E_pos"], [12, 3, 1, "", "W_K"], [12, 3, 1, "", "W_O"], [12, 3, 1, "", "W_Q"], [12, 3, 1, "", "W_U"], [12, 3, 1, "", "W_V"], [12, 3, 1, "", "W_in"], [12, 3, 1, "", "W_out"], [12, 3, 1, "", "W_pos"], [12, 2, 1, "", "all_head_labels"], [12, 3, 1, "", "b_K"], [12, 3, 1, "", "b_O"], [12, 3, 1, "", "b_Q"], [12, 3, 1, "", "b_U"], [12, 3, 1, "", "b_V"], [12, 3, 1, "", "b_in"], [12, 3, 1, "", "b_out"], [12, 2, 1, "", "cpu"], [12, 2, 1, "", "cuda"], [12, 2, 1, "", "forward"], [12, 2, 1, "", "from_pretrained"], [12, 2, 1, "", "mps"], [12, 2, 1, "", "run_with_cache"], [12, 2, 1, "", "to"]], "transformer_lens.HookedEncoderDecoder": [[13, 1, 1, "", "HookedEncoderDecoder"]], "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder": [[13, 3, 1, "", "OV"], [13, 3, 1, "", "QK"], [13, 3, 1, "", "W_E"], [13, 3, 1, "", "W_K"], [13, 3, 1, "", "W_O"], [13, 3, 1, "", "W_Q"], [13, 3, 1, "", "W_U"], [13, 3, 1, "", "W_V"], [13, 3, 1, "", "W_in"], [13, 3, 1, "", "W_out"], [13, 3, 1, "", "W_pos"], [13, 2, 1, "", "all_head_labels"], [13, 3, 1, "", "b_K"], [13, 3, 1, "", "b_O"], [13, 3, 1, "", "b_Q"], [13, 3, 1, "", "b_U"], [13, 3, 1, "", "b_V"], [13, 3, 1, "", "b_in"], [13, 3, 1, "", "b_out"], [13, 2, 1, "", "cpu"], [13, 2, 1, "", "cuda"], [13, 2, 1, "", "forward"], [13, 2, 1, "", "from_pretrained"], [13, 2, 1, "", "mps"], [13, 2, 1, "", "run_with_cache"], [13, 2, 1, "", "to"]], "transformer_lens.HookedTransformer": [[14, 1, 1, "", "HookedTransformer"], [14, 1, 1, "", "Output"]], "transformer_lens.HookedTransformer.HookedTransformer": [[14, 3, 1, "", "OV"], [14, 3, 1, "", "QK"], [14, 3, 1, "", "W_E"], [14, 3, 1, "", "W_E_pos"], [14, 3, 1, "", "W_K"], [14, 3, 1, "", "W_O"], [14, 3, 1, "", "W_Q"], [14, 3, 1, "", "W_U"], [14, 3, 1, "", "W_V"], [14, 3, 1, "", "W_gate"], [14, 3, 1, "", "W_in"], [14, 3, 1, "", "W_out"], [14, 3, 1, "", "W_pos"], [14, 2, 1, "", "__init__"], [14, 2, 1, "", "accumulated_bias"], [14, 2, 1, "", "all_composition_scores"], [14, 2, 1, "", "all_head_labels"], [14, 3, 1, "", "b_K"], [14, 3, 1, "", "b_O"], [14, 3, 1, "", "b_Q"], [14, 3, 1, "", "b_U"], [14, 3, 1, "", "b_V"], [14, 3, 1, "", "b_in"], [14, 3, 1, "", "b_out"], [14, 2, 1, "", "center_unembed"], [14, 2, 1, "", "center_writing_weights"], [14, 2, 1, "", "check_hooks_to_add"], [14, 2, 1, "", "cpu"], [14, 2, 1, "", "cuda"], [14, 2, 1, "", "fold_layer_norm"], [14, 2, 1, "", "fold_value_biases"], [14, 2, 1, "", "forward"], [14, 2, 1, "", "from_pretrained"], [14, 2, 1, "", "from_pretrained_no_processing"], [14, 2, 1, "", "generate"], [14, 2, 1, "", "get_token_position"], [14, 2, 1, "", "init_weights"], [14, 2, 1, "", "input_to_embed"], [14, 4, 1, "", "ln_final"], [14, 2, 1, "", "load_and_process_state_dict"], [14, 2, 1, "", "load_sample_training_dataset"], [14, 2, 1, "", "loss_fn"], [14, 2, 1, "", "move_model_modules_to_device"], [14, 2, 1, "", "mps"], [14, 2, 1, "", "process_weights_"], [14, 2, 1, "", "refactor_factored_attn_matrices"], [14, 2, 1, "", "run_with_cache"], [14, 2, 1, "", "sample_datapoint"], [14, 2, 1, "", "set_tokenizer"], [14, 2, 1, "", "set_use_attn_in"], [14, 2, 1, "", "set_use_attn_result"], [14, 2, 1, "", "set_use_hook_mlp_in"], [14, 2, 1, "", "set_use_split_qkv_input"], [14, 2, 1, "", "to"], [14, 2, 1, "", "to_single_str_token"], [14, 2, 1, "", "to_single_token"], [14, 2, 1, "", "to_str_tokens"], [14, 2, 1, "", "to_string"], [14, 2, 1, "", "to_tokens"], [14, 2, 1, "", "tokens_to_residual_directions"]], "transformer_lens.HookedTransformer.Output": [[14, 4, 1, "", "logits"], [14, 4, 1, "", "loss"]], "transformer_lens.HookedTransformerConfig": [[15, 1, 1, "", "HookedTransformerConfig"]], "transformer_lens.HookedTransformerConfig.HookedTransformerConfig": [[15, 4, 1, "", "act_fn"], [15, 4, 1, "", "attention_dir"], [15, 4, 1, "", "attn_only"], [15, 4, 1, "", "attn_scale"], [15, 4, 1, "", "attn_scores_soft_cap"], [15, 4, 1, "", "attn_types"], [15, 4, 1, "", "checkpoint_index"], [15, 4, 1, "", "checkpoint_label_type"], [15, 4, 1, "", "checkpoint_value"], [15, 4, 1, "", "d_head"], [15, 4, 1, "", "d_mlp"], [15, 4, 1, "", "d_model"], [15, 4, 1, "", "d_vocab"], [15, 4, 1, "", "d_vocab_out"], [15, 4, 1, "", "decoder_start_token_id"], [15, 4, 1, "", "default_prepend_bos"], [15, 4, 1, "", "device"], [15, 4, 1, "", "dtype"], [15, 4, 1, "", "eps"], [15, 4, 1, "", "experts_per_token"], [15, 4, 1, "", "final_rms"], [15, 4, 1, "", "from_checkpoint"], [15, 2, 1, "", "from_dict"], [15, 4, 1, "", "gated_mlp"], [15, 4, 1, "", "init_mode"], [15, 4, 1, "", "init_weights"], [15, 4, 1, "", "initializer_range"], [15, 2, 1, "", "is_layer_norm_activation"], [15, 4, 1, "", "load_in_4bit"], [15, 4, 1, "", "model_name"], [15, 4, 1, "", "n_ctx"], [15, 4, 1, "", "n_devices"], [15, 4, 1, "", "n_heads"], [15, 4, 1, "", "n_key_value_heads"], [15, 4, 1, "", "n_layers"], [15, 4, 1, "", "n_params"], [15, 4, 1, "", "normalization_type"], [15, 4, 1, "", "num_experts"], [15, 4, 1, "", "original_architecture"], [15, 4, 1, "", "output_logits_soft_cap"], [15, 4, 1, "", "parallel_attn_mlp"], [15, 4, 1, "", "positional_embedding_type"], [15, 4, 1, "", "post_embedding_ln"], [15, 4, 1, "", "relative_attention_max_distance"], [15, 4, 1, "", "relative_attention_num_buckets"], [15, 4, 1, "", "rotary_adjacent_pairs"], [15, 4, 1, "", "rotary_base"], [15, 4, 1, "", "rotary_dim"], [15, 4, 1, "", "scale_attn_by_inverse_layer_idx"], [15, 4, 1, "", "seed"], [15, 2, 1, "", "set_seed_everywhere"], [15, 4, 1, "", "tie_word_embeddings"], [15, 2, 1, "", "to_dict"], [15, 4, 1, "", "tokenizer_name"], [15, 4, 1, "", "tokenizer_prepends_bos"], [15, 4, 1, "", "trust_remote_code"], [15, 2, 1, "", "unwrap"], [15, 4, 1, "", "use_attn_in"], [15, 4, 1, "", "use_attn_result"], [15, 4, 1, "", "use_attn_scale"], [15, 4, 1, "", "use_hook_mlp_in"], [15, 4, 1, "", "use_hook_tokens"], [15, 4, 1, "", "use_local_attn"], [15, 4, 1, "", "use_normalization_before_and_after"], [15, 4, 1, "", "use_split_qkv_input"], [15, 4, 1, "", "window_size"]], "transformer_lens.SVDInterpreter": [[16, 1, 1, "", "SVDInterpreter"]], "transformer_lens.SVDInterpreter.SVDInterpreter": [[16, 2, 1, "", "get_singular_vectors"]], "transformer_lens.components": [[18, 0, 0, "-", "abstract_attention"], [19, 0, 0, "-", "attention"], [20, 0, 0, "-", "bert_block"], [21, 0, 0, "-", "bert_embed"], [22, 0, 0, "-", "bert_mlm_head"], [23, 0, 0, "-", "embed"], [24, 0, 0, "-", "grouped_query_attention"], [25, 0, 0, "-", "layer_norm"], [26, 0, 0, "-", "layer_norm_pre"], [27, 0, 0, "-", "pos_embed"], [28, 0, 0, "-", "rms_norm"], [29, 0, 0, "-", "rms_norm_pre"], [30, 0, 0, "-", "t5_attention"], [31, 0, 0, "-", "t5_block"], [32, 0, 0, "-", "token_typed_embed"], [33, 0, 0, "-", "transformer_block"], [34, 0, 0, "-", "unembed"]], "transformer_lens.components.abstract_attention": [[18, 1, 1, "", "AbstractAttention"]], "transformer_lens.components.abstract_attention.AbstractAttention": [[18, 3, 1, "", "OV"], [18, 3, 1, "", "QK"], [18, 2, 1, "", "__init__"], [18, 4, 1, "", "alibi"], [18, 2, 1, "", "apply_causal_mask"], [18, 2, 1, "", "apply_rotary"], [18, 2, 1, "", "calculate_attention_scores"], [18, 2, 1, "", "calculate_qkv_matrices"], [18, 2, 1, "", "calculate_sin_cos_rotary"], [18, 2, 1, "", "calculate_z_scores"], [18, 2, 1, "", "create_alibi_bias"], [18, 2, 1, "", "create_alibi_multipliers"], [18, 2, 1, "", "create_alibi_slope"], [18, 2, 1, "", "forward"], [18, 2, 1, "", "rotate_every_two"]], "transformer_lens.components.attention": [[19, 1, 1, "", "Attention"]], "transformer_lens.components.attention.Attention": [[19, 2, 1, "", "__init__"]], "transformer_lens.components.bert_block": [[20, 1, 1, "", "BertBlock"]], "transformer_lens.components.bert_block.BertBlock": [[20, 2, 1, "", "forward"]], "transformer_lens.components.bert_embed": [[21, 1, 1, "", "BertEmbed"]], "transformer_lens.components.bert_embed.BertEmbed": [[21, 2, 1, "", "forward"]], "transformer_lens.components.bert_mlm_head": [[22, 1, 1, "", "BertMLMHead"]], "transformer_lens.components.bert_mlm_head.BertMLMHead": [[22, 2, 1, "", "forward"]], "transformer_lens.components.embed": [[23, 1, 1, "", "Embed"]], "transformer_lens.components.embed.Embed": [[23, 2, 1, "", "forward"]], "transformer_lens.components.grouped_query_attention": [[24, 1, 1, "", "GroupedQueryAttention"]], "transformer_lens.components.grouped_query_attention.GroupedQueryAttention": [[24, 3, 1, "", "W_K"], [24, 3, 1, "", "W_V"], [24, 2, 1, "", "__init__"], [24, 3, 1, "", "b_K"], [24, 3, 1, "", "b_V"], [24, 2, 1, "", "calculate_attention_scores"], [24, 2, 1, "", "calculate_qkv_matrices"], [24, 2, 1, "", "calculate_z_scores"]], "transformer_lens.components.layer_norm": [[25, 1, 1, "", "LayerNorm"]], "transformer_lens.components.layer_norm.LayerNorm": [[25, 2, 1, "", "__init__"], [25, 2, 1, "", "forward"]], "transformer_lens.components.layer_norm_pre": [[26, 1, 1, "", "LayerNormPre"]], "transformer_lens.components.layer_norm_pre.LayerNormPre": [[26, 2, 1, "", "__init__"], [26, 2, 1, "", "forward"]], "transformer_lens.components.pos_embed": [[27, 1, 1, "", "PosEmbed"]], "transformer_lens.components.pos_embed.PosEmbed": [[27, 2, 1, "", "forward"]], "transformer_lens.components.rms_norm": [[28, 1, 1, "", "RMSNorm"]], "transformer_lens.components.rms_norm.RMSNorm": [[28, 2, 1, "", "__init__"], [28, 2, 1, "", "forward"]], "transformer_lens.components.rms_norm_pre": [[29, 1, 1, "", "RMSNormPre"]], "transformer_lens.components.rms_norm_pre.RMSNormPre": [[29, 2, 1, "", "__init__"], [29, 2, 1, "", "forward"]], "transformer_lens.components.t5_attention": [[30, 1, 1, "", "T5Attention"]], "transformer_lens.components.t5_attention.T5Attention": [[30, 2, 1, "", "compute_relative_attention_bias"]], "transformer_lens.components.t5_block": [[31, 1, 1, "", "T5Block"]], "transformer_lens.components.t5_block.T5Block": [[31, 2, 1, "", "forward"]], "transformer_lens.components.token_typed_embed": [[32, 1, 1, "", "TokenTypeEmbed"]], "transformer_lens.components.token_typed_embed.TokenTypeEmbed": [[32, 2, 1, "", "forward"]], "transformer_lens.components.transformer_block": [[33, 1, 1, "", "TransformerBlock"]], "transformer_lens.components.transformer_block.TransformerBlock": [[33, 2, 1, "", "apply_mlp"], [33, 2, 1, "", "forward"], [33, 4, 1, "", "ln1"], [33, 4, 1, "", "ln2"], [33, 4, 1, "", "mlp"]], "transformer_lens.components.unembed": [[34, 1, 1, "", "Unembed"]], "transformer_lens.components.unembed.Unembed": [[34, 2, 1, "", "forward"]], "transformer_lens.evals": [[35, 1, 1, "", "IOIDataset"], [35, 5, 1, "", "evaluate"], [35, 5, 1, "", "evaluate_on_dataset"], [35, 5, 1, "", "induction_loss"], [35, 5, 1, "", "ioi_eval"], [35, 5, 1, "", "make_code_data_loader"], [35, 5, 1, "", "make_owt_data_loader"], [35, 5, 1, "", "make_pile_data_loader"], [35, 5, 1, "", "make_wiki_data_loader"], [35, 5, 1, "", "sanity_check"]], "transformer_lens.evals.IOIDataset": [[35, 2, 1, "", "get_default_names"], [35, 2, 1, "", "get_default_nouns"], [35, 2, 1, "", "get_default_templates"], [35, 2, 1, "", "get_sample"]], "transformer_lens.head_detector": [[36, 5, 1, "", "compute_head_attention_similarity_score"], [36, 5, 1, "", "detect_head"], [36, 5, 1, "", "get_duplicate_token_head_detection_pattern"], [36, 5, 1, "", "get_induction_head_detection_pattern"], [36, 5, 1, "", "get_previous_token_head_detection_pattern"], [36, 5, 1, "", "get_supported_heads"]], "transformer_lens.hook_points": [[37, 4, 1, "", "HookFunction"], [37, 1, 1, "", "HookPoint"], [37, 1, 1, "", "HookedRootModule"], [37, 1, 1, "", "LensHandle"]], "transformer_lens.hook_points.HookPoint": [[37, 2, 1, "", "add_hook"], [37, 2, 1, "", "add_perma_hook"], [37, 2, 1, "", "clear_context"], [37, 2, 1, "", "forward"], [37, 2, 1, "", "layer"], [37, 2, 1, "", "remove_hooks"]], "transformer_lens.hook_points.HookedRootModule": [[37, 2, 1, "", "add_caching_hooks"], [37, 2, 1, "", "add_hook"], [37, 2, 1, "", "add_perma_hook"], [37, 2, 1, "", "cache_all"], [37, 2, 1, "", "cache_some"], [37, 2, 1, "", "check_and_add_hook"], [37, 2, 1, "", "check_hooks_to_add"], [37, 2, 1, "", "clear_contexts"], [37, 2, 1, "", "get_caching_hooks"], [37, 4, 1, "", "hook_dict"], [37, 2, 1, "", "hook_points"], [37, 2, 1, "", "hooks"], [37, 4, 1, "", "mod_dict"], [37, 4, 1, "", "name"], [37, 2, 1, "", "remove_all_hook_fns"], [37, 2, 1, "", "reset_hooks"], [37, 2, 1, "", "run_with_cache"], [37, 2, 1, "", "run_with_hooks"], [37, 2, 1, "", "setup"]], "transformer_lens.hook_points.LensHandle": [[37, 4, 1, "", "context_level"], [37, 4, 1, "", "hook"], [37, 4, 1, "", "is_permanent"]], "transformer_lens.loading_from_pretrained": [[38, 1, 1, "", "Config"], [38, 6, 1, "", "MODEL_ALIASES"], [38, 6, 1, "", "NON_HF_HOSTED_MODEL_NAMES"], [38, 6, 1, "", "OFFICIAL_MODEL_NAMES"], [38, 5, 1, "", "get_checkpoint_labels"], [38, 5, 1, "", "get_num_params_of_pretrained"], [38, 5, 1, "", "get_pretrained_model_config"]], "transformer_lens.loading_from_pretrained.Config": [[38, 4, 1, "", "d_head"], [38, 4, 1, "", "d_mlp"], [38, 4, 1, "", "d_model"], [38, 4, 1, "", "d_vocab"], [38, 4, 1, "", "debug"], [38, 4, 1, "", "init_range"], [38, 4, 1, "", "layer_norm_eps"], [38, 4, 1, "", "n_ctx"], [38, 4, 1, "", "n_heads"], [38, 4, 1, "", "n_layers"]], "transformer_lens.past_key_value_caching": [[39, 1, 1, "", "HookedTransformerKeyValueCache"], [39, 1, 1, "", "HookedTransformerKeyValueCacheEntry"]], "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCache": [[39, 2, 1, "", "append_attention_mask"], [39, 4, 1, "", "entries"], [39, 2, 1, "", "freeze"], [39, 4, 1, "", "frozen"], [39, 2, 1, "", "init_cache"], [39, 4, 1, "", "previous_attention_mask"], [39, 2, 1, "", "unfreeze"]], "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCacheEntry": [[39, 2, 1, "", "append"], [39, 4, 1, "", "frozen"], [39, 2, 1, "", "init_cache_entry"], [39, 4, 1, "", "past_keys"], [39, 4, 1, "", "past_values"]], "transformer_lens.patching": [[40, 5, 1, "", "generic_activation_patch"], [40, 5, 1, "", "get_act_patch_attn_head_all_pos_every"], [40, 5, 1, "", "get_act_patch_attn_head_by_pos_every"], [40, 5, 1, "", "get_act_patch_attn_head_k_all_pos"], [40, 5, 1, "", "get_act_patch_attn_head_k_by_pos"], [40, 5, 1, "", "get_act_patch_attn_head_out_all_pos"], [40, 5, 1, "", "get_act_patch_attn_head_out_by_pos"], [40, 5, 1, "", "get_act_patch_attn_head_pattern_all_pos"], [40, 5, 1, "", "get_act_patch_attn_head_pattern_by_pos"], [40, 5, 1, "", "get_act_patch_attn_head_pattern_dest_src_pos"], [40, 5, 1, "", "get_act_patch_attn_head_q_all_pos"], [40, 5, 1, "", "get_act_patch_attn_head_q_by_pos"], [40, 5, 1, "", "get_act_patch_attn_head_v_all_pos"], [40, 5, 1, "", "get_act_patch_attn_head_v_by_pos"], [40, 5, 1, "", "get_act_patch_attn_out"], [40, 5, 1, "", "get_act_patch_block_every"], [40, 5, 1, "", "get_act_patch_mlp_out"], [40, 5, 1, "", "get_act_patch_resid_mid"], [40, 5, 1, "", "get_act_patch_resid_pre"], [40, 5, 1, "", "layer_head_dest_src_pos_pattern_patch_setter"], [40, 5, 1, "", "layer_head_pattern_patch_setter"], [40, 5, 1, "", "layer_head_pos_pattern_patch_setter"], [40, 5, 1, "", "layer_head_vector_patch_setter"], [40, 5, 1, "", "layer_pos_head_vector_patch_setter"], [40, 5, 1, "", "layer_pos_patch_setter"]], "transformer_lens.pretrained.weight_conversions": [[43, 0, 0, "-", "bert"], [44, 0, 0, "-", "bloom"], [45, 0, 0, "-", "coder"], [46, 0, 0, "-", "gemma"], [47, 0, 0, "-", "gpt2"], [48, 0, 0, "-", "gptj"], [49, 0, 0, "-", "llama"], [50, 0, 0, "-", "mingpt"], [51, 0, 0, "-", "mistral"], [52, 0, 0, "-", "mixtral"], [53, 0, 0, "-", "nanogpt"], [54, 0, 0, "-", "neel_solu_old"], [55, 0, 0, "-", "neo"], [56, 0, 0, "-", "neox"], [57, 0, 0, "-", "opt"], [58, 0, 0, "-", "phi"], [59, 0, 0, "-", "phi3"], [60, 0, 0, "-", "qwen"], [61, 0, 0, "-", "qwen2"], [62, 0, 0, "-", "t5"]], "transformer_lens.pretrained.weight_conversions.bloom": [[44, 5, 1, "", "convert_bloom_weights"]], "transformer_lens.pretrained.weight_conversions.coder": [[45, 5, 1, "", "convert_coder_weights"]], "transformer_lens.pretrained.weight_conversions.mistral": [[51, 5, 1, "", "convert_mistral_weights"]], "transformer_lens.pretrained.weight_conversions.mixtral": [[52, 5, 1, "", "convert_mixtral_weights"]], "transformer_lens.pretrained.weight_conversions.phi": [[58, 5, 1, "", "convert_phi_weights"]], "transformer_lens.pretrained.weight_conversions.phi3": [[59, 5, 1, "", "convert_phi3_weights"]], "transformer_lens.pretrained.weight_conversions.qwen": [[60, 5, 1, "", "convert_qwen_weights"]], "transformer_lens.pretrained.weight_conversions.qwen2": [[61, 5, 1, "", "convert_qwen2_weights"]], "transformer_lens.pretrained.weight_conversions.t5": [[62, 5, 1, "", "convert_t5_weights"]], "transformer_lens.train": [[63, 1, 1, "", "HookedTransformerTrainConfig"], [63, 5, 1, "", "train"]], "transformer_lens.train.HookedTransformerTrainConfig": [[63, 4, 1, "", "batch_size"], [63, 4, 1, "", "device"], [63, 4, 1, "", "lr"], [63, 4, 1, "", "max_grad_norm"], [63, 4, 1, "", "max_steps"], [63, 4, 1, "", "momentum"], [63, 4, 1, "", "num_epochs"], [63, 4, 1, "", "optimizer_name"], [63, 4, 1, "", "print_every"], [63, 4, 1, "", "save_dir"], [63, 4, 1, "", "save_every"], [63, 4, 1, "", "seed"], [63, 4, 1, "", "wandb"], [63, 4, 1, "", "wandb_project_name"], [63, 4, 1, "", "warmup_steps"], [63, 4, 1, "", "weight_decay"]], "transformer_lens.utilities": [[65, 0, 0, "-", "activation_functions"], [66, 0, 0, "-", "addmm"], [67, 0, 0, "-", "attention"], [68, 0, 0, "-", "devices"]], "transformer_lens.utilities.addmm": [[66, 5, 1, "", "batch_addmm"], [66, 5, 1, "", "vanilla_addmm"]], "transformer_lens.utilities.attention": [[67, 5, 1, "", "complex_attn_linear"], [67, 5, 1, "", "simple_attn_linear"]], "transformer_lens.utilities.devices": [[68, 5, 1, "", "get_device_for_block_index"], [68, 5, 1, "", "move_to_and_update_config"]], "transformer_lens.utils": [[69, 1, 1, "", "LocallyOverridenDefaults"], [69, 1, 1, "", "Slice"], [69, 6, 1, "", "SliceInput"], [69, 5, 1, "", "calc_fan_in_and_fan_out"], [69, 5, 1, "", "composition_scores"], [69, 5, 1, "", "download_file_from_hf"], [69, 5, 1, "", "gelu_fast"], [69, 5, 1, "", "gelu_new"], [69, 5, 1, "", "get_act_name"], [69, 5, 1, "", "get_attention_mask"], [69, 5, 1, "", "get_corner"], [69, 5, 1, "", "get_cumsum_along_dim"], [69, 5, 1, "", "get_dataset"], [69, 5, 1, "", "get_device"], [69, 5, 1, "", "get_input_with_manually_prepended_bos"], [69, 5, 1, "", "get_nested_attr"], [69, 5, 1, "", "get_offset_position_ids"], [69, 5, 1, "", "get_tokenizer_with_bos"], [69, 5, 1, "", "get_tokens_with_bos_removed"], [69, 5, 1, "", "init_kaiming_normal_"], [69, 5, 1, "", "init_kaiming_uniform_"], [69, 5, 1, "", "init_xavier_normal_"], [69, 5, 1, "", "init_xavier_uniform_"], [69, 5, 1, "", "is_lower_triangular"], [69, 5, 1, "", "is_square"], [69, 5, 1, "", "keep_single_column"], [69, 5, 1, "", "lm_accuracy"], [69, 5, 1, "", "lm_cross_entropy_loss"], [69, 5, 1, "", "override_or_use_default_value"], [69, 5, 1, "", "print_gpu_mem"], [69, 5, 1, "", "remove_batch_dim"], [69, 5, 1, "", "repeat_along_head_dimension"], [69, 5, 1, "", "sample_logits"], [69, 5, 1, "", "set_nested_attr"], [69, 5, 1, "", "solu"], [69, 5, 1, "", "test_prompt"], [69, 5, 1, "", "to_numpy"], [69, 5, 1, "", "tokenize_and_concatenate"], [69, 5, 1, "", "transpose"]], "transformer_lens.utils.LocallyOverridenDefaults": [[69, 2, 1, "", "__init__"]], "transformer_lens.utils.Slice": [[69, 2, 1, "", "__init__"], [69, 2, 1, "", "apply"], [69, 2, 1, "", "indices"], [69, 4, 1, "", "slice"], [69, 2, 1, "", "unwrap"]]}, "objtypes": {"0": "py:module", "1": "py:class", "2": "py:method", "3": "py:property", "4": "py:attribute", "5": "py:function", "6": "py:data"}, "objnames": {"0": ["py", "module", "Python module"], "1": ["py", "class", "Python class"], "2": ["py", "method", "Python method"], "3": ["py", "property", "Python property"], "4": ["py", "attribute", "Python attribute"], "5": ["py", "function", "Python function"], "6": ["py", "data", "Python data"]}, "titleterms": {"citat": 0, "contribut": 1, "setup": [1, 70, 71], "devcontain": 1, "manual": 1, "test": [1, 5], "run": [1, 71], "format": 1, "document": 1, "docstr": 1, "style": 1, "guid": 1, "section": 1, "order": 1, "support": 1, "sphinx": 1, "properti": [1, 72], "refer": 1, "other": [1, 71], "function": [1, 70], "class": [1, 71], "math": 1, "markup": 1, "galleri": 2, "get": [3, 4], "start": [3, 4, 7], "advic": 3, "read": [3, 70], "code": 3, "instal": 3, "huggingfac": 3, "gate": 3, "access": [3, 71], "mechanist": [4, 73], "interpret": [4, 71, 73], "transformerlen": [5, 73], "2": 5, "0": 5, "first": 5, "an": [5, 71], "introduct": [5, 70, 71], "adopt": 5, "semant": 5, "version": 5, "deprec": 5, "roadmap": 5, "immedi": 5, "within": 5, "next": 5, "month": 5, "mid": 5, "term": 5, "3": 5, "perform": 5, "streamlin": 5, "ad": 5, "new": 5, "model": [5, 71, 72, 73], "long": 5, "year": 5, "integr": 5, "contributor": 5, "dev": 5, "branch": 5, "coverag": 5, "compon": [5, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34], "refactor": 5, "conclus": 5, "appendix": 5, "special": 6, "case": 6, "mixtur": 6, "expert": 6, "error": 6, "rate": 6, "tutori": 7, "where": 7, "To": 7, "demo": [7, 70, 71], "transform": [8, 71], "len": [8, 70, 71], "api": 8, "content": 8, "transformer_len": [9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69], "submodul": [9, 17, 42, 64], "subpackag": [9, 41], "activationcach": 10, "factoredmatrix": 11, "hookedencod": 12, "hookedencoderdecod": 13, "hookedtransform": 14, "hookedtransformerconfig": 15, "svdinterpret": 16, "abstract_attent": 18, "attent": [19, 67, 70], "bert_block": 20, "bert_emb": 21, "bert_mlm_head": 22, "emb": 23, "grouped_query_attent": 24, "layer_norm": 25, "layer_norm_pr": 26, "pos_emb": 27, "rms_norm": 28, "rms_norm_pr": 29, "t5_attent": 30, "t5_block": 31, "token_typed_emb": 32, "transformer_block": 33, "unemb": 34, "eval": 35, "head_detector": 36, "hook_point": 37, "loading_from_pretrain": 38, "past_key_value_cach": 39, "patch": [40, 70, 71], "pretrain": [41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62], "weight_convers": [42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62], "bert": 43, "bloom": 44, "coder": 45, "gemma": 46, "gpt2": 47, "gptj": 48, "llama": 49, "mingpt": 50, "mistral": 51, "mixtral": 52, "nanogpt": 53, "neel_solu_old": 54, "neo": 55, "neox": 56, "opt": 57, "phi": 58, "phi3": 59, "qwen": 60, "qwen2": 61, "t5": 62, "train": [63, 71], "util": [64, 65, 66, 67, 68, 69], "activation_funct": 65, "addmm": 66, "devic": 68, "exploratori": 70, "analysi": 70, "tip": 70, "thi": 70, "environ": 70, "ignor": 70, "import": [70, 71], "pytorch": 70, "plot": 70, "helper": 70, "indirect": [70, 71], "object": [70, 71], "identif": [70, 71], "brainstorm": 70, "what": 70, "": 70, "actual": 70, "go": 70, "On": 70, "option": 70, "direct": 70, "logit": 70, "attribut": 70, "layer": 70, "head": [70, 71], "activ": [70, 71], "residu": 70, "stream": 70, "decompos": 70, "consolid": 70, "understand": 70, "visual": 70, "pattern": 70, "compar": 70, "paper": 70, "bonu": 70, "explor": 70, "anomali": 70, "earli": 70, "ar": 70, "induct": [70, 71], "implic": 70, "backup": 70, "name": [70, 71], "mover": 70, "main": 71, "notebook": 71, "load": 71, "cach": 71, "all": 71, "hook": 71, "interven": 71, "task": 71, "avail": 71, "overview": 71, "open": 71, "sourc": 71, "librari": [71, 73], "some": 71, "friendli": 71, "i": 71, "ve": 71, "includ": 71, "resourc": 71, "architectur": 71, "paramet": 71, "fold": 71, "layernorm": 71, "For": 71, "curiou": 71, "featur": 71, "deal": 71, "token": 71, "gotcha": 71, "prepend_bo": 71, "factor": 71, "matrix": 71, "basic": 71, "exampl": 71, "medium": 71, "eigenvalu": 71, "copi": 71, "score": 71, "gener": [71, 73], "text": 71, "point": 71, "toi": 71, "pre": 71, "checkpoint": 71, "phase": 71, "transit": 71, "tabl": 72, "A": 73, "languag": 73}, "envversion": {"sphinx.domains.c": 2, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 8, "sphinx.domains.index": 1, "sphinx.domains.javascript": 2, "sphinx.domains.math": 2, "sphinx.domains.python": 3, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "nbsphinx": 4, "sphinx": 57}, "alltitles": {"Citation": [[0, "citation"]], "Contributing": [[1, "contributing"]], "Setup": [[1, "setup"], [70, "Setup"], [71, "Setup"]], "DevContainer": [[1, "devcontainer"]], "Manual Setup": [[1, "manual-setup"]], "Testing": [[1, "testing"]], "Running the tests": [[1, "running-the-tests"]], "Formatting": [[1, "formatting"]], "Documentation": [[1, "documentation"]], "Docstring Style Guide": [[1, "docstring-style-guide"]], "Sections and Order": [[1, "sections-and-order"]], "Supported Sphinx Properties": [[1, "supported-sphinx-properties"]], "References to Other Functions/Classes": [[1, "references-to-other-functions-classes"]], "Maths": [[1, "maths"]], "Markup": [[1, "markup"]], "Gallery": [[2, "gallery"]], "Getting Started": [[3, "getting-started"]], "Advice for Reading the Code": [[3, "advice-for-reading-the-code"]], "Installation": [[3, "installation"]], "Huggingface Gated Access": [[3, "huggingface-gated-access"]], "Getting Started in Mechanistic Interpretability": [[4, "getting-started-in-mechanistic-interpretability"]], "TransformerLens 2.0": [[5, "transformerlens-2-0"]], "First, an introduction": [[5, "first-an-introduction"]], "Adopting Semantic Versioning": [[5, "adopting-semantic-versioning"]], "Deprecations": [[5, "deprecations"]], "Roadmap": [[5, "roadmap"]], "Immediate - within the next month": [[5, "immediate-within-the-next-month"]], "Mid-term - within the next 3 months": [[5, "mid-term-within-the-next-3-months"]], "Performance": [[5, "performance"]], "Streamlining Adding New Models": [[5, "streamlining-adding-new-models"]], "Long-term - within the next year": [[5, "long-term-within-the-next-year"]], "Model Testing": [[5, "model-testing"]], "Model Integration": [[5, "model-integration"]], "Contributors": [[5, "contributors"]], "New Dev Branches": [[5, "new-dev-branches"]], "Integration Tests": [[5, "integration-tests"]], "Test Coverage": [[5, "test-coverage"]], "Components Refactor": [[5, "components-refactor"]], "Conclusion": [[5, "conclusion"]], "Appendix": [[5, "appendix"]], "Semantic Versioning": [[5, "semantic-versioning"]], "Special Cases": [[6, "special-cases"]], "Mixture of Experts error rates": [[6, "mixture-of-experts-error-rates"]], "Tutorials": [[7, "tutorials"]], "Where To Start": [[7, "where-to-start"]], "Demos": [[7, "demos"]], "Transformer Lens API": [[8, "transformer-lens-api"]], "Contents": [[8, "contents"]], "transformer_lens": [[9, "transformer-lens"]], "Submodules": [[9, "submodules"], [17, "submodules"], [42, "submodules"], [64, "submodules"]], "Subpackages": [[9, "subpackages"], [41, "subpackages"]], "transformer_lens.ActivationCache": [[10, "module-transformer_lens.ActivationCache"]], "transformer_lens.FactoredMatrix": [[11, "module-transformer_lens.FactoredMatrix"]], "transformer_lens.HookedEncoder": [[12, "module-transformer_lens.HookedEncoder"]], "transformer_lens.HookedEncoderDecoder": [[13, "module-transformer_lens.HookedEncoderDecoder"]], "transformer_lens.HookedTransformer": [[14, "module-transformer_lens.HookedTransformer"]], "transformer_lens.HookedTransformerConfig": [[15, "module-transformer_lens.HookedTransformerConfig"]], "transformer_lens.SVDInterpreter": [[16, "module-transformer_lens.SVDInterpreter"]], "transformer_lens.components": [[17, "transformer-lens-components"]], "transformer_lens.components.abstract_attention": [[18, "module-transformer_lens.components.abstract_attention"]], "transformer_lens.components.attention": [[19, "module-transformer_lens.components.attention"]], "transformer_lens.components.bert_block": [[20, "module-transformer_lens.components.bert_block"]], "transformer_lens.components.bert_embed": [[21, "module-transformer_lens.components.bert_embed"]], "transformer_lens.components.bert_mlm_head": [[22, "module-transformer_lens.components.bert_mlm_head"]], "transformer_lens.components.embed": [[23, "module-transformer_lens.components.embed"]], "transformer_lens.components.grouped_query_attention": [[24, "module-transformer_lens.components.grouped_query_attention"]], "transformer_lens.components.layer_norm": [[25, "module-transformer_lens.components.layer_norm"]], "transformer_lens.components.layer_norm_pre": [[26, "module-transformer_lens.components.layer_norm_pre"]], "transformer_lens.components.pos_embed": [[27, "module-transformer_lens.components.pos_embed"]], "transformer_lens.components.rms_norm": [[28, "module-transformer_lens.components.rms_norm"]], "transformer_lens.components.rms_norm_pre": [[29, "module-transformer_lens.components.rms_norm_pre"]], "transformer_lens.components.t5_attention": [[30, "module-transformer_lens.components.t5_attention"]], "transformer_lens.components.t5_block": [[31, "module-transformer_lens.components.t5_block"]], "transformer_lens.components.token_typed_embed": [[32, "module-transformer_lens.components.token_typed_embed"]], "transformer_lens.components.transformer_block": [[33, "module-transformer_lens.components.transformer_block"]], "transformer_lens.components.unembed": [[34, "module-transformer_lens.components.unembed"]], "transformer_lens.evals": [[35, "module-transformer_lens.evals"]], "transformer_lens.head_detector": [[36, "module-transformer_lens.head_detector"]], "transformer_lens.hook_points": [[37, "module-transformer_lens.hook_points"]], "transformer_lens.loading_from_pretrained": [[38, "module-transformer_lens.loading_from_pretrained"]], "transformer_lens.past_key_value_caching": [[39, "module-transformer_lens.past_key_value_caching"]], "transformer_lens.patching": [[40, "module-transformer_lens.patching"]], "transformer_lens.pretrained": [[41, "transformer-lens-pretrained"]], "transformer_lens.pretrained.weight_conversions": [[42, "transformer-lens-pretrained-weight-conversions"]], "transformer_lens.pretrained.weight_conversions.bert": [[43, "module-transformer_lens.pretrained.weight_conversions.bert"]], "transformer_lens.pretrained.weight_conversions.bloom": [[44, "module-transformer_lens.pretrained.weight_conversions.bloom"]], "transformer_lens.pretrained.weight_conversions.coder": [[45, "module-transformer_lens.pretrained.weight_conversions.coder"]], "transformer_lens.pretrained.weight_conversions.gemma": [[46, "module-transformer_lens.pretrained.weight_conversions.gemma"]], "transformer_lens.pretrained.weight_conversions.gpt2": [[47, "module-transformer_lens.pretrained.weight_conversions.gpt2"]], "transformer_lens.pretrained.weight_conversions.gptj": [[48, "module-transformer_lens.pretrained.weight_conversions.gptj"]], "transformer_lens.pretrained.weight_conversions.llama": [[49, "module-transformer_lens.pretrained.weight_conversions.llama"]], "transformer_lens.pretrained.weight_conversions.mingpt": [[50, "module-transformer_lens.pretrained.weight_conversions.mingpt"]], "transformer_lens.pretrained.weight_conversions.mistral": [[51, "module-transformer_lens.pretrained.weight_conversions.mistral"]], "transformer_lens.pretrained.weight_conversions.mixtral": [[52, "module-transformer_lens.pretrained.weight_conversions.mixtral"]], "transformer_lens.pretrained.weight_conversions.nanogpt": [[53, "module-transformer_lens.pretrained.weight_conversions.nanogpt"]], "transformer_lens.pretrained.weight_conversions.neel_solu_old": [[54, "module-transformer_lens.pretrained.weight_conversions.neel_solu_old"]], "transformer_lens.pretrained.weight_conversions.neo": [[55, "module-transformer_lens.pretrained.weight_conversions.neo"]], "transformer_lens.pretrained.weight_conversions.neox": [[56, "module-transformer_lens.pretrained.weight_conversions.neox"]], "transformer_lens.pretrained.weight_conversions.opt": [[57, "module-transformer_lens.pretrained.weight_conversions.opt"]], "transformer_lens.pretrained.weight_conversions.phi": [[58, "module-transformer_lens.pretrained.weight_conversions.phi"]], "transformer_lens.pretrained.weight_conversions.phi3": [[59, "module-transformer_lens.pretrained.weight_conversions.phi3"]], "transformer_lens.pretrained.weight_conversions.qwen": [[60, "module-transformer_lens.pretrained.weight_conversions.qwen"]], "transformer_lens.pretrained.weight_conversions.qwen2": [[61, "module-transformer_lens.pretrained.weight_conversions.qwen2"]], "transformer_lens.pretrained.weight_conversions.t5": [[62, "module-transformer_lens.pretrained.weight_conversions.t5"]], "transformer_lens.train": [[63, "module-transformer_lens.train"]], "transformer_lens.utilities": [[64, "transformer-lens-utilities"]], "transformer_lens.utilities.activation_functions": [[65, "module-transformer_lens.utilities.activation_functions"]], "transformer_lens.utilities.addmm": [[66, "module-transformer_lens.utilities.addmm"]], "transformer_lens.utilities.attention": [[67, "module-transformer_lens.utilities.attention"]], "transformer_lens.utilities.devices": [[68, "module-transformer_lens.utilities.devices"]], "transformer_lens.utils": [[69, "module-transformer_lens.utils"]], "Exploratory Analysis Demo": [[70, "Exploratory-Analysis-Demo"]], "Tips for Reading This": [[70, "Tips-for-Reading-This"]], "Environment Setup (ignore)": [[70, "Environment-Setup-(ignore)"]], "Imports": [[70, "Imports"]], "PyTorch Setup": [[70, "PyTorch-Setup"]], "Plotting Helper Functions (ignore)": [[70, "Plotting-Helper-Functions-(ignore)"]], "Introduction": [[70, "Introduction"], [71, "Introduction"]], "Indirect Object Identification": [[70, "Indirect-Object-Identification"]], "Brainstorm What\u2019s Actually Going On (Optional)": [[70, "Brainstorm-What's-Actually-Going-On-(Optional)"]], "Direct Logit Attribution": [[70, "Direct-Logit-Attribution"]], "Logit Lens": [[70, "Logit-Lens"]], "Layer Attribution": [[70, "Layer-Attribution"]], "Head Attribution": [[70, "Head-Attribution"]], "Attention Analysis": [[70, "Attention-Analysis"]], "Activation Patching": [[70, "Activation-Patching"]], "Residual Stream": [[70, "Residual-Stream"]], "Layers": [[70, "Layers"]], "Heads": [[70, "Heads"]], "Decomposing Heads": [[70, "Decomposing-Heads"]], "Consolidating Understanding": [[70, "Consolidating-Understanding"]], "Visualizing Attention Patterns": [[70, "Visualizing-Attention-Patterns"]], "Comparing to the Paper": [[70, "Comparing-to-the-Paper"]], "Bonus: Exploring Anomalies": [[70, "Bonus:-Exploring-Anomalies"]], "Early Heads are Induction Heads(?!)": [[70, "Early-Heads-are-Induction-Heads(?!)"]], "Implications": [[70, "Implications"]], "Backup Name Mover Heads": [[70, "Backup-Name-Mover-Heads"]], "Transformer Lens Main Demo Notebook": [[71, "Transformer-Lens-Main-Demo-Notebook"]], "Loading and Running Models": [[71, "Loading-and-Running-Models"]], "Caching all Activations": [[71, "Caching-all-Activations"]], "Hooks: Intervening on Activations": [[71, "Hooks:-Intervening-on-Activations"]], "Activation Patching on the Indirect Object Identification Task": [[71, "Activation-Patching-on-the-Indirect-Object-Identification-Task"]], "Hooks: Accessing Activations": [[71, "Hooks:-Accessing-Activations"]], "Available Models": [[71, "Available-Models"]], "An overview of the important open source models in the library": [[71, "An-overview-of-the-important-open-source-models-in-the-library"]], "An overview of some interpretability-friendly models I\u2019ve trained and included": [[71, "An-overview-of-some-interpretability-friendly-models-I've-trained-and-included"]], "Other Resources:": [[71, "Other-Resources:"]], "Transformer architecture": [[71, "Transformer-architecture"]], "Parameter Names": [[71, "Parameter-Names"]], "Activation + Hook Names": [[71, "Activation-+-Hook-Names"]], "Folding LayerNorm (For the Curious)": [[71, "Folding-LayerNorm-(For-the-Curious)"]], "Features": [[71, "Features"]], "Dealing with tokens": [[71, "Dealing-with-tokens"]], "Gotcha: prepend_bos": [[71, "Gotcha:-prepend_bos"]], "Factored Matrix Class": [[71, "Factored-Matrix-Class"]], "Basic Examples": [[71, "Basic-Examples"]], "Medium Example: Eigenvalue Copying Scores": [[71, "Medium-Example:-Eigenvalue-Copying-Scores"]], "Generating Text": [[71, "Generating-Text"]], "Hook Points": [[71, "Hook-Points"]], "Toy Example": [[71, "Toy-Example"]], "Loading Pre-Trained Checkpoints": [[71, "Loading-Pre-Trained-Checkpoints"]], "Example: Induction Head Phase Transition": [[71, "Example:-Induction-Head-Phase-Transition"]], "Model Properties Table": [[72, "model-properties-table"]], "TransformerLens": [[73, "transformerlens"]], "A Library for Mechanistic Interpretability of Generative Language Models": [[73, "a-library-for-mechanistic-interpretability-of-generative-language-models"]]}, "indexentries": {"activationcache (class in transformer_lens.activationcache)": [[10, "transformer_lens.ActivationCache.ActivationCache"]], "accumulated_resid() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.accumulated_resid"]], "apply_ln_to_stack() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.apply_ln_to_stack"]], "apply_slice_to_batch_dim() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.apply_slice_to_batch_dim"]], "compute_head_results() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.compute_head_results"]], "decompose_resid() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.decompose_resid"]], "get_full_resid_decomposition() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.get_full_resid_decomposition"]], "get_neuron_results() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.get_neuron_results"]], "items() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.items"]], "keys() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.keys"]], "logit_attrs() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.logit_attrs"]], "module": [[10, "module-transformer_lens.ActivationCache"], [11, "module-transformer_lens.FactoredMatrix"], [12, "module-transformer_lens.HookedEncoder"], [13, "module-transformer_lens.HookedEncoderDecoder"], [14, "module-transformer_lens.HookedTransformer"], [15, "module-transformer_lens.HookedTransformerConfig"], [16, "module-transformer_lens.SVDInterpreter"], [18, "module-transformer_lens.components.abstract_attention"], [19, "module-transformer_lens.components.attention"], [20, "module-transformer_lens.components.bert_block"], [21, "module-transformer_lens.components.bert_embed"], [22, "module-transformer_lens.components.bert_mlm_head"], [23, "module-transformer_lens.components.embed"], [24, "module-transformer_lens.components.grouped_query_attention"], [25, "module-transformer_lens.components.layer_norm"], [26, "module-transformer_lens.components.layer_norm_pre"], [27, "module-transformer_lens.components.pos_embed"], [28, "module-transformer_lens.components.rms_norm"], [29, "module-transformer_lens.components.rms_norm_pre"], [30, "module-transformer_lens.components.t5_attention"], [31, "module-transformer_lens.components.t5_block"], [32, "module-transformer_lens.components.token_typed_embed"], [33, "module-transformer_lens.components.transformer_block"], [34, "module-transformer_lens.components.unembed"], [35, "module-transformer_lens.evals"], [36, "module-transformer_lens.head_detector"], [37, "module-transformer_lens.hook_points"], [38, "module-transformer_lens.loading_from_pretrained"], [39, "module-transformer_lens.past_key_value_caching"], [40, "module-transformer_lens.patching"], [43, "module-transformer_lens.pretrained.weight_conversions.bert"], [44, "module-transformer_lens.pretrained.weight_conversions.bloom"], [45, "module-transformer_lens.pretrained.weight_conversions.coder"], [46, "module-transformer_lens.pretrained.weight_conversions.gemma"], [47, "module-transformer_lens.pretrained.weight_conversions.gpt2"], [48, "module-transformer_lens.pretrained.weight_conversions.gptj"], [49, "module-transformer_lens.pretrained.weight_conversions.llama"], [50, "module-transformer_lens.pretrained.weight_conversions.mingpt"], [51, "module-transformer_lens.pretrained.weight_conversions.mistral"], [52, "module-transformer_lens.pretrained.weight_conversions.mixtral"], [53, "module-transformer_lens.pretrained.weight_conversions.nanogpt"], [54, "module-transformer_lens.pretrained.weight_conversions.neel_solu_old"], [55, "module-transformer_lens.pretrained.weight_conversions.neo"], [56, "module-transformer_lens.pretrained.weight_conversions.neox"], [57, "module-transformer_lens.pretrained.weight_conversions.opt"], [58, "module-transformer_lens.pretrained.weight_conversions.phi"], [59, "module-transformer_lens.pretrained.weight_conversions.phi3"], [60, "module-transformer_lens.pretrained.weight_conversions.qwen"], [61, "module-transformer_lens.pretrained.weight_conversions.qwen2"], [62, "module-transformer_lens.pretrained.weight_conversions.t5"], [63, "module-transformer_lens.train"], [65, "module-transformer_lens.utilities.activation_functions"], [66, "module-transformer_lens.utilities.addmm"], [67, "module-transformer_lens.utilities.attention"], [68, "module-transformer_lens.utilities.devices"], [69, "module-transformer_lens.utils"]], "remove_batch_dim() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.remove_batch_dim"]], "stack_activation() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.stack_activation"]], "stack_head_results() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.stack_head_results"]], "stack_neuron_results() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.stack_neuron_results"]], "to() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.to"]], "toggle_autodiff() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.toggle_autodiff"]], "transformer_lens.activationcache": [[10, "module-transformer_lens.ActivationCache"]], "values() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.values"]], "ab (transformer_lens.factoredmatrix.factoredmatrix property)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.AB"]], "ba (transformer_lens.factoredmatrix.factoredmatrix property)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.BA"]], "factoredmatrix (class in transformer_lens.factoredmatrix)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix"]], "s (transformer_lens.factoredmatrix.factoredmatrix property)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.S"]], "t (transformer_lens.factoredmatrix.factoredmatrix property)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.T"]], "u (transformer_lens.factoredmatrix.factoredmatrix property)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.U"]], "vh (transformer_lens.factoredmatrix.factoredmatrix property)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.Vh"]], "collapse_l() (transformer_lens.factoredmatrix.factoredmatrix method)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.collapse_l"]], "collapse_r() (transformer_lens.factoredmatrix.factoredmatrix method)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.collapse_r"]], "eigenvalues (transformer_lens.factoredmatrix.factoredmatrix property)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.eigenvalues"]], "get_corner() (transformer_lens.factoredmatrix.factoredmatrix method)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.get_corner"]], "make_even() (transformer_lens.factoredmatrix.factoredmatrix method)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.make_even"]], "ndim (transformer_lens.factoredmatrix.factoredmatrix property)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.ndim"]], "norm() (transformer_lens.factoredmatrix.factoredmatrix method)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.norm"]], "pair (transformer_lens.factoredmatrix.factoredmatrix property)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.pair"]], "svd() (transformer_lens.factoredmatrix.factoredmatrix method)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.svd"]], "transformer_lens.factoredmatrix": [[11, "module-transformer_lens.FactoredMatrix"]], "unsqueeze() (transformer_lens.factoredmatrix.factoredmatrix method)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.unsqueeze"]], "hookedencoder (class in transformer_lens.hookedencoder)": [[12, "transformer_lens.HookedEncoder.HookedEncoder"]], "ov (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.OV"]], "qk (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.QK"]], "w_e (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_E"]], "w_e_pos (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_E_pos"]], "w_k (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_K"]], "w_o (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_O"]], "w_q (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_Q"]], "w_u (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_U"]], "w_v (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_V"]], "w_in (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_in"]], "w_out (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_out"]], "w_pos (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_pos"]], "all_head_labels() (transformer_lens.hookedencoder.hookedencoder method)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.all_head_labels"]], "b_k (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.b_K"]], "b_o (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.b_O"]], "b_q (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.b_Q"]], "b_u (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.b_U"]], "b_v (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.b_V"]], "b_in (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.b_in"]], "b_out (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.b_out"]], "cpu() (transformer_lens.hookedencoder.hookedencoder method)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.cpu"]], "cuda() (transformer_lens.hookedencoder.hookedencoder method)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.cuda"]], "forward() (transformer_lens.hookedencoder.hookedencoder method)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.forward"]], "from_pretrained() (transformer_lens.hookedencoder.hookedencoder class method)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.from_pretrained"]], "mps() (transformer_lens.hookedencoder.hookedencoder method)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.mps"]], "run_with_cache() (transformer_lens.hookedencoder.hookedencoder method)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.run_with_cache"]], "to() (transformer_lens.hookedencoder.hookedencoder method)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.to"]], "transformer_lens.hookedencoder": [[12, "module-transformer_lens.HookedEncoder"]], "hookedencoderdecoder (class in transformer_lens.hookedencoderdecoder)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder"]], "ov (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.OV"]], "qk (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.QK"]], "w_e (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.W_E"]], "w_k (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.W_K"]], "w_o (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.W_O"]], "w_q (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.W_Q"]], "w_u (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.W_U"]], "w_v (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.W_V"]], "w_in (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.W_in"]], "w_out (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.W_out"]], "w_pos (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.W_pos"]], "all_head_labels() (transformer_lens.hookedencoderdecoder.hookedencoderdecoder method)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.all_head_labels"]], "b_k (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.b_K"]], "b_o (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.b_O"]], "b_q (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.b_Q"]], "b_u (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.b_U"]], "b_v (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.b_V"]], "b_in (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.b_in"]], "b_out (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.b_out"]], "cpu() (transformer_lens.hookedencoderdecoder.hookedencoderdecoder method)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.cpu"]], "cuda() (transformer_lens.hookedencoderdecoder.hookedencoderdecoder method)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.cuda"]], "forward() (transformer_lens.hookedencoderdecoder.hookedencoderdecoder method)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.forward"]], "from_pretrained() (transformer_lens.hookedencoderdecoder.hookedencoderdecoder class method)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.from_pretrained"]], "mps() (transformer_lens.hookedencoderdecoder.hookedencoderdecoder method)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.mps"]], "run_with_cache() (transformer_lens.hookedencoderdecoder.hookedencoderdecoder method)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.run_with_cache"]], "to() (transformer_lens.hookedencoderdecoder.hookedencoderdecoder method)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.to"]], "transformer_lens.hookedencoderdecoder": [[13, "module-transformer_lens.HookedEncoderDecoder"]], "hookedtransformer (class in transformer_lens.hookedtransformer)": [[14, "transformer_lens.HookedTransformer.HookedTransformer"]], "ov (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.OV"]], "output (class in transformer_lens.hookedtransformer)": [[14, "transformer_lens.HookedTransformer.Output"]], "qk (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.QK"]], "w_e (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_E"]], "w_e_pos (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_E_pos"]], "w_k (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_K"]], "w_o (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_O"]], "w_q (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_Q"]], "w_u (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_U"]], "w_v (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_V"]], "w_gate (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_gate"]], "w_in (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_in"]], "w_out (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_out"]], "w_pos (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_pos"]], "__init__() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.__init__"]], "accumulated_bias() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.accumulated_bias"]], "all_composition_scores() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.all_composition_scores"]], "all_head_labels() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.all_head_labels"]], "b_k (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.b_K"]], "b_o (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.b_O"]], "b_q (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.b_Q"]], "b_u (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.b_U"]], "b_v (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.b_V"]], "b_in (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.b_in"]], "b_out (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.b_out"]], "center_unembed() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.center_unembed"]], "center_writing_weights() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.center_writing_weights"]], "check_hooks_to_add() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.check_hooks_to_add"]], "cpu() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.cpu"]], "cuda() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.cuda"]], "fold_layer_norm() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.fold_layer_norm"]], "fold_value_biases() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.fold_value_biases"]], "forward() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.forward"]], "from_pretrained() (transformer_lens.hookedtransformer.hookedtransformer class method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.from_pretrained"]], "from_pretrained_no_processing() (transformer_lens.hookedtransformer.hookedtransformer class method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.from_pretrained_no_processing"]], "generate() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.generate"]], "get_token_position() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.get_token_position"]], "init_weights() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.init_weights"]], "input_to_embed() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.input_to_embed"]], "ln_final (transformer_lens.hookedtransformer.hookedtransformer attribute)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.ln_final"]], "load_and_process_state_dict() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.load_and_process_state_dict"]], "load_sample_training_dataset() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.load_sample_training_dataset"]], "logits (transformer_lens.hookedtransformer.output attribute)": [[14, "transformer_lens.HookedTransformer.Output.logits"]], "loss (transformer_lens.hookedtransformer.output attribute)": [[14, "transformer_lens.HookedTransformer.Output.loss"]], "loss_fn() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.loss_fn"]], "move_model_modules_to_device() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.move_model_modules_to_device"]], "mps() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.mps"]], "process_weights_() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.process_weights_"]], "refactor_factored_attn_matrices() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.refactor_factored_attn_matrices"]], "run_with_cache() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.run_with_cache"]], "sample_datapoint() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.sample_datapoint"]], "set_tokenizer() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.set_tokenizer"]], "set_use_attn_in() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.set_use_attn_in"]], "set_use_attn_result() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.set_use_attn_result"]], "set_use_hook_mlp_in() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.set_use_hook_mlp_in"]], "set_use_split_qkv_input() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.set_use_split_qkv_input"]], "to() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.to"]], "to_single_str_token() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.to_single_str_token"]], "to_single_token() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.to_single_token"]], "to_str_tokens() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.to_str_tokens"]], "to_string() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.to_string"]], "to_tokens() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.to_tokens"]], "tokens_to_residual_directions() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.tokens_to_residual_directions"]], "transformer_lens.hookedtransformer": [[14, "module-transformer_lens.HookedTransformer"]], "hookedtransformerconfig (class in transformer_lens.hookedtransformerconfig)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig"]], "act_fn (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.act_fn"]], "attention_dir (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.attention_dir"]], "attn_only (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.attn_only"]], "attn_scale (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.attn_scale"]], "attn_scores_soft_cap (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.attn_scores_soft_cap"]], "attn_types (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.attn_types"]], "checkpoint_index (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.checkpoint_index"]], "checkpoint_label_type (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.checkpoint_label_type"]], "checkpoint_value (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.checkpoint_value"]], "d_head (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.d_head"]], "d_mlp (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.d_mlp"]], "d_model (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.d_model"]], "d_vocab (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.d_vocab"]], "d_vocab_out (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.d_vocab_out"]], "decoder_start_token_id (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.decoder_start_token_id"]], "default_prepend_bos (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.default_prepend_bos"]], "device (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.device"]], "dtype (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.dtype"]], "eps (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.eps"]], "experts_per_token (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.experts_per_token"]], "final_rms (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.final_rms"]], "from_checkpoint (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.from_checkpoint"]], "from_dict() (transformer_lens.hookedtransformerconfig.hookedtransformerconfig class method)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.from_dict"]], "gated_mlp (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.gated_mlp"]], "init_mode (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.init_mode"]], "init_weights (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.init_weights"]], "initializer_range (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.initializer_range"]], "is_layer_norm_activation() (transformer_lens.hookedtransformerconfig.hookedtransformerconfig method)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.is_layer_norm_activation"]], "load_in_4bit (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.load_in_4bit"]], "model_name (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.model_name"]], "n_ctx (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.n_ctx"]], "n_devices (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.n_devices"]], "n_heads (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.n_heads"]], "n_key_value_heads (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.n_key_value_heads"]], "n_layers (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.n_layers"]], "n_params (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.n_params"]], "normalization_type (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.normalization_type"]], "num_experts (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.num_experts"]], "original_architecture (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.original_architecture"]], "output_logits_soft_cap (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.output_logits_soft_cap"]], "parallel_attn_mlp (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.parallel_attn_mlp"]], "positional_embedding_type (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.positional_embedding_type"]], "post_embedding_ln (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.post_embedding_ln"]], "relative_attention_max_distance (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.relative_attention_max_distance"]], "relative_attention_num_buckets (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.relative_attention_num_buckets"]], "rotary_adjacent_pairs (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.rotary_adjacent_pairs"]], "rotary_base (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.rotary_base"]], "rotary_dim (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.rotary_dim"]], "scale_attn_by_inverse_layer_idx (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.scale_attn_by_inverse_layer_idx"]], "seed (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.seed"]], "set_seed_everywhere() (transformer_lens.hookedtransformerconfig.hookedtransformerconfig method)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.set_seed_everywhere"]], "tie_word_embeddings (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.tie_word_embeddings"]], "to_dict() (transformer_lens.hookedtransformerconfig.hookedtransformerconfig method)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.to_dict"]], "tokenizer_name (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.tokenizer_name"]], "tokenizer_prepends_bos (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.tokenizer_prepends_bos"]], "transformer_lens.hookedtransformerconfig": [[15, "module-transformer_lens.HookedTransformerConfig"]], "trust_remote_code (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.trust_remote_code"]], "unwrap() (transformer_lens.hookedtransformerconfig.hookedtransformerconfig class method)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.unwrap"]], "use_attn_in (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.use_attn_in"]], "use_attn_result (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.use_attn_result"]], "use_attn_scale (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.use_attn_scale"]], "use_hook_mlp_in (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.use_hook_mlp_in"]], "use_hook_tokens (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.use_hook_tokens"]], "use_local_attn (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.use_local_attn"]], "use_normalization_before_and_after (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.use_normalization_before_and_after"]], "use_split_qkv_input (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.use_split_qkv_input"]], "window_size (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.window_size"]], "svdinterpreter (class in transformer_lens.svdinterpreter)": [[16, "transformer_lens.SVDInterpreter.SVDInterpreter"]], "get_singular_vectors() (transformer_lens.svdinterpreter.svdinterpreter method)": [[16, "transformer_lens.SVDInterpreter.SVDInterpreter.get_singular_vectors"]], "transformer_lens.svdinterpreter": [[16, "module-transformer_lens.SVDInterpreter"]], "abstractattention (class in transformer_lens.components.abstract_attention)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention"]], "ov (transformer_lens.components.abstract_attention.abstractattention property)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.OV"]], "qk (transformer_lens.components.abstract_attention.abstractattention property)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.QK"]], "__init__() (transformer_lens.components.abstract_attention.abstractattention method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.__init__"]], "alibi (transformer_lens.components.abstract_attention.abstractattention attribute)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.alibi"]], "apply_causal_mask() (transformer_lens.components.abstract_attention.abstractattention method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.apply_causal_mask"]], "apply_rotary() (transformer_lens.components.abstract_attention.abstractattention method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.apply_rotary"]], "calculate_attention_scores() (transformer_lens.components.abstract_attention.abstractattention method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.calculate_attention_scores"]], "calculate_qkv_matrices() (transformer_lens.components.abstract_attention.abstractattention method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.calculate_qkv_matrices"]], "calculate_sin_cos_rotary() (transformer_lens.components.abstract_attention.abstractattention method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.calculate_sin_cos_rotary"]], "calculate_z_scores() (transformer_lens.components.abstract_attention.abstractattention method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.calculate_z_scores"]], "create_alibi_bias() (transformer_lens.components.abstract_attention.abstractattention static method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.create_alibi_bias"]], "create_alibi_multipliers() (transformer_lens.components.abstract_attention.abstractattention static method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.create_alibi_multipliers"]], "create_alibi_slope() (transformer_lens.components.abstract_attention.abstractattention static method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.create_alibi_slope"]], "forward() (transformer_lens.components.abstract_attention.abstractattention method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.forward"]], "rotate_every_two() (transformer_lens.components.abstract_attention.abstractattention method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.rotate_every_two"]], "transformer_lens.components.abstract_attention": [[18, "module-transformer_lens.components.abstract_attention"]], "attention (class in transformer_lens.components.attention)": [[19, "transformer_lens.components.attention.Attention"]], "__init__() (transformer_lens.components.attention.attention method)": [[19, "transformer_lens.components.attention.Attention.__init__"]], "transformer_lens.components.attention": [[19, "module-transformer_lens.components.attention"]], "bertblock (class in transformer_lens.components.bert_block)": [[20, "transformer_lens.components.bert_block.BertBlock"]], "forward() (transformer_lens.components.bert_block.bertblock method)": [[20, "transformer_lens.components.bert_block.BertBlock.forward"]], "transformer_lens.components.bert_block": [[20, "module-transformer_lens.components.bert_block"]], "bertembed (class in transformer_lens.components.bert_embed)": [[21, "transformer_lens.components.bert_embed.BertEmbed"]], "forward() (transformer_lens.components.bert_embed.bertembed method)": [[21, "transformer_lens.components.bert_embed.BertEmbed.forward"]], "transformer_lens.components.bert_embed": [[21, "module-transformer_lens.components.bert_embed"]], "bertmlmhead (class in transformer_lens.components.bert_mlm_head)": [[22, "transformer_lens.components.bert_mlm_head.BertMLMHead"]], "forward() (transformer_lens.components.bert_mlm_head.bertmlmhead method)": [[22, "transformer_lens.components.bert_mlm_head.BertMLMHead.forward"]], "transformer_lens.components.bert_mlm_head": [[22, "module-transformer_lens.components.bert_mlm_head"]], "embed (class in transformer_lens.components.embed)": [[23, "transformer_lens.components.embed.Embed"]], "forward() (transformer_lens.components.embed.embed method)": [[23, "transformer_lens.components.embed.Embed.forward"]], "transformer_lens.components.embed": [[23, "module-transformer_lens.components.embed"]], "groupedqueryattention (class in transformer_lens.components.grouped_query_attention)": [[24, "transformer_lens.components.grouped_query_attention.GroupedQueryAttention"]], "w_k (transformer_lens.components.grouped_query_attention.groupedqueryattention property)": [[24, "transformer_lens.components.grouped_query_attention.GroupedQueryAttention.W_K"]], "w_v (transformer_lens.components.grouped_query_attention.groupedqueryattention property)": [[24, "transformer_lens.components.grouped_query_attention.GroupedQueryAttention.W_V"]], "__init__() (transformer_lens.components.grouped_query_attention.groupedqueryattention method)": [[24, "transformer_lens.components.grouped_query_attention.GroupedQueryAttention.__init__"]], "b_k (transformer_lens.components.grouped_query_attention.groupedqueryattention property)": [[24, "transformer_lens.components.grouped_query_attention.GroupedQueryAttention.b_K"]], "b_v (transformer_lens.components.grouped_query_attention.groupedqueryattention property)": [[24, "transformer_lens.components.grouped_query_attention.GroupedQueryAttention.b_V"]], "calculate_attention_scores() (transformer_lens.components.grouped_query_attention.groupedqueryattention method)": [[24, "transformer_lens.components.grouped_query_attention.GroupedQueryAttention.calculate_attention_scores"]], "calculate_qkv_matrices() (transformer_lens.components.grouped_query_attention.groupedqueryattention method)": [[24, "transformer_lens.components.grouped_query_attention.GroupedQueryAttention.calculate_qkv_matrices"]], "calculate_z_scores() (transformer_lens.components.grouped_query_attention.groupedqueryattention method)": [[24, "transformer_lens.components.grouped_query_attention.GroupedQueryAttention.calculate_z_scores"]], "transformer_lens.components.grouped_query_attention": [[24, "module-transformer_lens.components.grouped_query_attention"]], "layernorm (class in transformer_lens.components.layer_norm)": [[25, "transformer_lens.components.layer_norm.LayerNorm"]], "__init__() (transformer_lens.components.layer_norm.layernorm method)": [[25, "transformer_lens.components.layer_norm.LayerNorm.__init__"]], "forward() (transformer_lens.components.layer_norm.layernorm method)": [[25, "transformer_lens.components.layer_norm.LayerNorm.forward"]], "transformer_lens.components.layer_norm": [[25, "module-transformer_lens.components.layer_norm"]], "layernormpre (class in transformer_lens.components.layer_norm_pre)": [[26, "transformer_lens.components.layer_norm_pre.LayerNormPre"]], "__init__() (transformer_lens.components.layer_norm_pre.layernormpre method)": [[26, "transformer_lens.components.layer_norm_pre.LayerNormPre.__init__"]], "forward() (transformer_lens.components.layer_norm_pre.layernormpre method)": [[26, "transformer_lens.components.layer_norm_pre.LayerNormPre.forward"]], "transformer_lens.components.layer_norm_pre": [[26, "module-transformer_lens.components.layer_norm_pre"]], "posembed (class in transformer_lens.components.pos_embed)": [[27, "transformer_lens.components.pos_embed.PosEmbed"]], "forward() (transformer_lens.components.pos_embed.posembed method)": [[27, "transformer_lens.components.pos_embed.PosEmbed.forward"]], "transformer_lens.components.pos_embed": [[27, "module-transformer_lens.components.pos_embed"]], "rmsnorm (class in transformer_lens.components.rms_norm)": [[28, "transformer_lens.components.rms_norm.RMSNorm"]], "__init__() (transformer_lens.components.rms_norm.rmsnorm method)": [[28, "transformer_lens.components.rms_norm.RMSNorm.__init__"]], "forward() (transformer_lens.components.rms_norm.rmsnorm method)": [[28, "transformer_lens.components.rms_norm.RMSNorm.forward"]], "transformer_lens.components.rms_norm": [[28, "module-transformer_lens.components.rms_norm"]], "rmsnormpre (class in transformer_lens.components.rms_norm_pre)": [[29, "transformer_lens.components.rms_norm_pre.RMSNormPre"]], "__init__() (transformer_lens.components.rms_norm_pre.rmsnormpre method)": [[29, "transformer_lens.components.rms_norm_pre.RMSNormPre.__init__"]], "forward() (transformer_lens.components.rms_norm_pre.rmsnormpre method)": [[29, "transformer_lens.components.rms_norm_pre.RMSNormPre.forward"]], "transformer_lens.components.rms_norm_pre": [[29, "module-transformer_lens.components.rms_norm_pre"]], "t5attention (class in transformer_lens.components.t5_attention)": [[30, "transformer_lens.components.t5_attention.T5Attention"]], "compute_relative_attention_bias() (transformer_lens.components.t5_attention.t5attention method)": [[30, "transformer_lens.components.t5_attention.T5Attention.compute_relative_attention_bias"]], "transformer_lens.components.t5_attention": [[30, "module-transformer_lens.components.t5_attention"]], "t5block (class in transformer_lens.components.t5_block)": [[31, "transformer_lens.components.t5_block.T5Block"]], "forward() (transformer_lens.components.t5_block.t5block method)": [[31, "transformer_lens.components.t5_block.T5Block.forward"]], "transformer_lens.components.t5_block": [[31, "module-transformer_lens.components.t5_block"]], "tokentypeembed (class in transformer_lens.components.token_typed_embed)": [[32, "transformer_lens.components.token_typed_embed.TokenTypeEmbed"]], "forward() (transformer_lens.components.token_typed_embed.tokentypeembed method)": [[32, "transformer_lens.components.token_typed_embed.TokenTypeEmbed.forward"]], "transformer_lens.components.token_typed_embed": [[32, "module-transformer_lens.components.token_typed_embed"]], "transformerblock (class in transformer_lens.components.transformer_block)": [[33, "transformer_lens.components.transformer_block.TransformerBlock"]], "apply_mlp() (transformer_lens.components.transformer_block.transformerblock method)": [[33, "transformer_lens.components.transformer_block.TransformerBlock.apply_mlp"]], "forward() (transformer_lens.components.transformer_block.transformerblock method)": [[33, "transformer_lens.components.transformer_block.TransformerBlock.forward"]], "ln1 (transformer_lens.components.transformer_block.transformerblock attribute)": [[33, "transformer_lens.components.transformer_block.TransformerBlock.ln1"]], "ln2 (transformer_lens.components.transformer_block.transformerblock attribute)": [[33, "transformer_lens.components.transformer_block.TransformerBlock.ln2"]], "mlp (transformer_lens.components.transformer_block.transformerblock attribute)": [[33, "transformer_lens.components.transformer_block.TransformerBlock.mlp"]], "transformer_lens.components.transformer_block": [[33, "module-transformer_lens.components.transformer_block"]], "unembed (class in transformer_lens.components.unembed)": [[34, "transformer_lens.components.unembed.Unembed"]], "forward() (transformer_lens.components.unembed.unembed method)": [[34, "transformer_lens.components.unembed.Unembed.forward"]], "transformer_lens.components.unembed": [[34, "module-transformer_lens.components.unembed"]], "ioidataset (class in transformer_lens.evals)": [[35, "transformer_lens.evals.IOIDataset"]], "evaluate() (in module transformer_lens.evals)": [[35, "transformer_lens.evals.evaluate"]], "evaluate_on_dataset() (in module transformer_lens.evals)": [[35, "transformer_lens.evals.evaluate_on_dataset"]], "get_default_names() (transformer_lens.evals.ioidataset static method)": [[35, "transformer_lens.evals.IOIDataset.get_default_names"]], "get_default_nouns() (transformer_lens.evals.ioidataset static method)": [[35, "transformer_lens.evals.IOIDataset.get_default_nouns"]], "get_default_templates() (transformer_lens.evals.ioidataset static method)": [[35, "transformer_lens.evals.IOIDataset.get_default_templates"]], "get_sample() (transformer_lens.evals.ioidataset method)": [[35, "transformer_lens.evals.IOIDataset.get_sample"]], "induction_loss() (in module transformer_lens.evals)": [[35, "transformer_lens.evals.induction_loss"]], "ioi_eval() (in module transformer_lens.evals)": [[35, "transformer_lens.evals.ioi_eval"]], "make_code_data_loader() (in module transformer_lens.evals)": [[35, "transformer_lens.evals.make_code_data_loader"]], "make_owt_data_loader() (in module transformer_lens.evals)": [[35, "transformer_lens.evals.make_owt_data_loader"]], "make_pile_data_loader() (in module transformer_lens.evals)": [[35, "transformer_lens.evals.make_pile_data_loader"]], "make_wiki_data_loader() (in module transformer_lens.evals)": [[35, "transformer_lens.evals.make_wiki_data_loader"]], "sanity_check() (in module transformer_lens.evals)": [[35, "transformer_lens.evals.sanity_check"]], "transformer_lens.evals": [[35, "module-transformer_lens.evals"]], "compute_head_attention_similarity_score() (in module transformer_lens.head_detector)": [[36, "transformer_lens.head_detector.compute_head_attention_similarity_score"]], "detect_head() (in module transformer_lens.head_detector)": [[36, "transformer_lens.head_detector.detect_head"]], "get_duplicate_token_head_detection_pattern() (in module transformer_lens.head_detector)": [[36, "transformer_lens.head_detector.get_duplicate_token_head_detection_pattern"]], "get_induction_head_detection_pattern() (in module transformer_lens.head_detector)": [[36, "transformer_lens.head_detector.get_induction_head_detection_pattern"]], "get_previous_token_head_detection_pattern() (in module transformer_lens.head_detector)": [[36, "transformer_lens.head_detector.get_previous_token_head_detection_pattern"]], "get_supported_heads() (in module transformer_lens.head_detector)": [[36, "transformer_lens.head_detector.get_supported_heads"]], "transformer_lens.head_detector": [[36, "module-transformer_lens.head_detector"]], "hookfunction (in module transformer_lens.hook_points)": [[37, "transformer_lens.hook_points.HookFunction"]], "hookpoint (class in transformer_lens.hook_points)": [[37, "transformer_lens.hook_points.HookPoint"]], "hookedrootmodule (class in transformer_lens.hook_points)": [[37, "transformer_lens.hook_points.HookedRootModule"]], "lenshandle (class in transformer_lens.hook_points)": [[37, "transformer_lens.hook_points.LensHandle"]], "add_caching_hooks() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.add_caching_hooks"]], "add_hook() (transformer_lens.hook_points.hookpoint method)": [[37, "transformer_lens.hook_points.HookPoint.add_hook"]], "add_hook() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.add_hook"]], "add_perma_hook() (transformer_lens.hook_points.hookpoint method)": [[37, "transformer_lens.hook_points.HookPoint.add_perma_hook"]], "add_perma_hook() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.add_perma_hook"]], "cache_all() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.cache_all"]], "cache_some() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.cache_some"]], "check_and_add_hook() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.check_and_add_hook"]], "check_hooks_to_add() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.check_hooks_to_add"]], "clear_context() (transformer_lens.hook_points.hookpoint method)": [[37, "transformer_lens.hook_points.HookPoint.clear_context"]], "clear_contexts() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.clear_contexts"]], "context_level (transformer_lens.hook_points.lenshandle attribute)": [[37, "transformer_lens.hook_points.LensHandle.context_level"]], "forward() (transformer_lens.hook_points.hookpoint method)": [[37, "transformer_lens.hook_points.HookPoint.forward"]], "get_caching_hooks() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.get_caching_hooks"]], "hook (transformer_lens.hook_points.lenshandle attribute)": [[37, "transformer_lens.hook_points.LensHandle.hook"]], "hook_dict (transformer_lens.hook_points.hookedrootmodule attribute)": [[37, "transformer_lens.hook_points.HookedRootModule.hook_dict"]], "hook_points() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.hook_points"]], "hooks() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.hooks"]], "is_permanent (transformer_lens.hook_points.lenshandle attribute)": [[37, "transformer_lens.hook_points.LensHandle.is_permanent"]], "layer() (transformer_lens.hook_points.hookpoint method)": [[37, "transformer_lens.hook_points.HookPoint.layer"]], "mod_dict (transformer_lens.hook_points.hookedrootmodule attribute)": [[37, "transformer_lens.hook_points.HookedRootModule.mod_dict"]], "name (transformer_lens.hook_points.hookedrootmodule attribute)": [[37, "transformer_lens.hook_points.HookedRootModule.name"]], "remove_all_hook_fns() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.remove_all_hook_fns"]], "remove_hooks() (transformer_lens.hook_points.hookpoint method)": [[37, "transformer_lens.hook_points.HookPoint.remove_hooks"]], "reset_hooks() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.reset_hooks"]], "run_with_cache() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.run_with_cache"]], "run_with_hooks() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.run_with_hooks"]], "setup() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.setup"]], "transformer_lens.hook_points": [[37, "module-transformer_lens.hook_points"]], "config (class in transformer_lens.loading_from_pretrained)": [[38, "transformer_lens.loading_from_pretrained.Config"]], "model_aliases (in module transformer_lens.loading_from_pretrained)": [[38, "transformer_lens.loading_from_pretrained.MODEL_ALIASES"]], "non_hf_hosted_model_names (in module transformer_lens.loading_from_pretrained)": [[38, "transformer_lens.loading_from_pretrained.NON_HF_HOSTED_MODEL_NAMES"]], "official_model_names (in module transformer_lens.loading_from_pretrained)": [[38, "transformer_lens.loading_from_pretrained.OFFICIAL_MODEL_NAMES"]], "d_head (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.d_head"]], "d_mlp (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.d_mlp"]], "d_model (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.d_model"]], "d_vocab (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.d_vocab"]], "debug (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.debug"]], "get_checkpoint_labels() (in module transformer_lens.loading_from_pretrained)": [[38, "transformer_lens.loading_from_pretrained.get_checkpoint_labels"]], "get_num_params_of_pretrained() (in module transformer_lens.loading_from_pretrained)": [[38, "transformer_lens.loading_from_pretrained.get_num_params_of_pretrained"]], "get_pretrained_model_config() (in module transformer_lens.loading_from_pretrained)": [[38, "transformer_lens.loading_from_pretrained.get_pretrained_model_config"]], "init_range (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.init_range"]], "layer_norm_eps (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.layer_norm_eps"]], "n_ctx (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.n_ctx"]], "n_heads (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.n_heads"]], "n_layers (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.n_layers"]], "transformer_lens.loading_from_pretrained": [[38, "module-transformer_lens.loading_from_pretrained"]], "hookedtransformerkeyvaluecache (class in transformer_lens.past_key_value_caching)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCache"]], "hookedtransformerkeyvaluecacheentry (class in transformer_lens.past_key_value_caching)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCacheEntry"]], "append() (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecacheentry method)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCacheEntry.append"]], "append_attention_mask() (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecache method)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCache.append_attention_mask"]], "entries (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecache attribute)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCache.entries"]], "freeze() (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecache method)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCache.freeze"]], "frozen (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecache attribute)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCache.frozen"]], "frozen (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecacheentry attribute)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCacheEntry.frozen"]], "init_cache() (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecache class method)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCache.init_cache"]], "init_cache_entry() (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecacheentry class method)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCacheEntry.init_cache_entry"]], "past_keys (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecacheentry attribute)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCacheEntry.past_keys"]], "past_values (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecacheentry attribute)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCacheEntry.past_values"]], "previous_attention_mask (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecache attribute)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCache.previous_attention_mask"]], "transformer_lens.past_key_value_caching": [[39, "module-transformer_lens.past_key_value_caching"]], "unfreeze() (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecache method)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCache.unfreeze"]], "generic_activation_patch() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.generic_activation_patch"]], "get_act_patch_attn_head_all_pos_every() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_all_pos_every"]], "get_act_patch_attn_head_by_pos_every() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_by_pos_every"]], "get_act_patch_attn_head_k_all_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_k_all_pos"]], "get_act_patch_attn_head_k_by_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_k_by_pos"]], "get_act_patch_attn_head_out_all_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_out_all_pos"]], "get_act_patch_attn_head_out_by_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_out_by_pos"]], "get_act_patch_attn_head_pattern_all_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_pattern_all_pos"]], "get_act_patch_attn_head_pattern_by_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_pattern_by_pos"]], "get_act_patch_attn_head_pattern_dest_src_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_pattern_dest_src_pos"]], "get_act_patch_attn_head_q_all_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_q_all_pos"]], "get_act_patch_attn_head_q_by_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_q_by_pos"]], "get_act_patch_attn_head_v_all_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_v_all_pos"]], "get_act_patch_attn_head_v_by_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_v_by_pos"]], "get_act_patch_attn_out() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_out"]], "get_act_patch_block_every() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_block_every"]], "get_act_patch_mlp_out() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_mlp_out"]], "get_act_patch_resid_mid() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_resid_mid"]], "get_act_patch_resid_pre() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_resid_pre"]], "layer_head_dest_src_pos_pattern_patch_setter() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.layer_head_dest_src_pos_pattern_patch_setter"]], "layer_head_pattern_patch_setter() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.layer_head_pattern_patch_setter"]], "layer_head_pos_pattern_patch_setter() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.layer_head_pos_pattern_patch_setter"]], "layer_head_vector_patch_setter() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.layer_head_vector_patch_setter"]], "layer_pos_head_vector_patch_setter() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.layer_pos_head_vector_patch_setter"]], "layer_pos_patch_setter() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.layer_pos_patch_setter"]], "transformer_lens.patching": [[40, "module-transformer_lens.patching"]], "transformer_lens.pretrained.weight_conversions.bert": [[43, "module-transformer_lens.pretrained.weight_conversions.bert"]], "convert_bloom_weights() (in module transformer_lens.pretrained.weight_conversions.bloom)": [[44, "transformer_lens.pretrained.weight_conversions.bloom.convert_bloom_weights"]], "transformer_lens.pretrained.weight_conversions.bloom": [[44, "module-transformer_lens.pretrained.weight_conversions.bloom"]], "convert_coder_weights() (in module transformer_lens.pretrained.weight_conversions.coder)": [[45, "transformer_lens.pretrained.weight_conversions.coder.convert_coder_weights"]], "transformer_lens.pretrained.weight_conversions.coder": [[45, "module-transformer_lens.pretrained.weight_conversions.coder"]], "transformer_lens.pretrained.weight_conversions.gemma": [[46, "module-transformer_lens.pretrained.weight_conversions.gemma"]], "transformer_lens.pretrained.weight_conversions.gpt2": [[47, "module-transformer_lens.pretrained.weight_conversions.gpt2"]], "transformer_lens.pretrained.weight_conversions.gptj": [[48, "module-transformer_lens.pretrained.weight_conversions.gptj"]], "transformer_lens.pretrained.weight_conversions.llama": [[49, "module-transformer_lens.pretrained.weight_conversions.llama"]], "transformer_lens.pretrained.weight_conversions.mingpt": [[50, "module-transformer_lens.pretrained.weight_conversions.mingpt"]], "convert_mistral_weights() (in module transformer_lens.pretrained.weight_conversions.mistral)": [[51, "transformer_lens.pretrained.weight_conversions.mistral.convert_mistral_weights"]], "transformer_lens.pretrained.weight_conversions.mistral": [[51, "module-transformer_lens.pretrained.weight_conversions.mistral"]], "convert_mixtral_weights() (in module transformer_lens.pretrained.weight_conversions.mixtral)": [[52, "transformer_lens.pretrained.weight_conversions.mixtral.convert_mixtral_weights"]], "transformer_lens.pretrained.weight_conversions.mixtral": [[52, "module-transformer_lens.pretrained.weight_conversions.mixtral"]], "transformer_lens.pretrained.weight_conversions.nanogpt": [[53, "module-transformer_lens.pretrained.weight_conversions.nanogpt"]], "transformer_lens.pretrained.weight_conversions.neel_solu_old": [[54, "module-transformer_lens.pretrained.weight_conversions.neel_solu_old"]], "transformer_lens.pretrained.weight_conversions.neo": [[55, "module-transformer_lens.pretrained.weight_conversions.neo"]], "transformer_lens.pretrained.weight_conversions.neox": [[56, "module-transformer_lens.pretrained.weight_conversions.neox"]], "transformer_lens.pretrained.weight_conversions.opt": [[57, "module-transformer_lens.pretrained.weight_conversions.opt"]], "convert_phi_weights() (in module transformer_lens.pretrained.weight_conversions.phi)": [[58, "transformer_lens.pretrained.weight_conversions.phi.convert_phi_weights"]], "transformer_lens.pretrained.weight_conversions.phi": [[58, "module-transformer_lens.pretrained.weight_conversions.phi"]], "convert_phi3_weights() (in module transformer_lens.pretrained.weight_conversions.phi3)": [[59, "transformer_lens.pretrained.weight_conversions.phi3.convert_phi3_weights"]], "transformer_lens.pretrained.weight_conversions.phi3": [[59, "module-transformer_lens.pretrained.weight_conversions.phi3"]], "convert_qwen_weights() (in module transformer_lens.pretrained.weight_conversions.qwen)": [[60, "transformer_lens.pretrained.weight_conversions.qwen.convert_qwen_weights"]], "transformer_lens.pretrained.weight_conversions.qwen": [[60, "module-transformer_lens.pretrained.weight_conversions.qwen"]], "convert_qwen2_weights() (in module transformer_lens.pretrained.weight_conversions.qwen2)": [[61, "transformer_lens.pretrained.weight_conversions.qwen2.convert_qwen2_weights"]], "transformer_lens.pretrained.weight_conversions.qwen2": [[61, "module-transformer_lens.pretrained.weight_conversions.qwen2"]], "convert_t5_weights() (in module transformer_lens.pretrained.weight_conversions.t5)": [[62, "transformer_lens.pretrained.weight_conversions.t5.convert_t5_weights"]], "transformer_lens.pretrained.weight_conversions.t5": [[62, "module-transformer_lens.pretrained.weight_conversions.t5"]], "hookedtransformertrainconfig (class in transformer_lens.train)": [[63, "transformer_lens.train.HookedTransformerTrainConfig"]], "batch_size (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.batch_size"]], "device (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.device"]], "lr (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.lr"]], "max_grad_norm (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.max_grad_norm"]], "max_steps (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.max_steps"]], "momentum (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.momentum"]], "num_epochs (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.num_epochs"]], "optimizer_name (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.optimizer_name"]], "print_every (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.print_every"]], "save_dir (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.save_dir"]], "save_every (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.save_every"]], "seed (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.seed"]], "train() (in module transformer_lens.train)": [[63, "transformer_lens.train.train"]], "transformer_lens.train": [[63, "module-transformer_lens.train"]], "wandb (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.wandb"]], "wandb_project_name (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.wandb_project_name"]], "warmup_steps (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.warmup_steps"]], "weight_decay (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.weight_decay"]], "transformer_lens.utilities.activation_functions": [[65, "module-transformer_lens.utilities.activation_functions"]], "batch_addmm() (in module transformer_lens.utilities.addmm)": [[66, "transformer_lens.utilities.addmm.batch_addmm"]], "transformer_lens.utilities.addmm": [[66, "module-transformer_lens.utilities.addmm"]], "vanilla_addmm() (in module transformer_lens.utilities.addmm)": [[66, "transformer_lens.utilities.addmm.vanilla_addmm"]], "complex_attn_linear() (in module transformer_lens.utilities.attention)": [[67, "transformer_lens.utilities.attention.complex_attn_linear"]], "simple_attn_linear() (in module transformer_lens.utilities.attention)": [[67, "transformer_lens.utilities.attention.simple_attn_linear"]], "transformer_lens.utilities.attention": [[67, "module-transformer_lens.utilities.attention"]], "get_device_for_block_index() (in module transformer_lens.utilities.devices)": [[68, "transformer_lens.utilities.devices.get_device_for_block_index"]], "move_to_and_update_config() (in module transformer_lens.utilities.devices)": [[68, "transformer_lens.utilities.devices.move_to_and_update_config"]], "transformer_lens.utilities.devices": [[68, "module-transformer_lens.utilities.devices"]], "locallyoverridendefaults (class in transformer_lens.utils)": [[69, "transformer_lens.utils.LocallyOverridenDefaults"]], "slice (class in transformer_lens.utils)": [[69, "transformer_lens.utils.Slice"]], "sliceinput (in module transformer_lens.utils)": [[69, "transformer_lens.utils.SliceInput"]], "__init__() (transformer_lens.utils.locallyoverridendefaults method)": [[69, "transformer_lens.utils.LocallyOverridenDefaults.__init__"]], "__init__() (transformer_lens.utils.slice method)": [[69, "transformer_lens.utils.Slice.__init__"]], "apply() (transformer_lens.utils.slice method)": [[69, "transformer_lens.utils.Slice.apply"]], "calc_fan_in_and_fan_out() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.calc_fan_in_and_fan_out"]], "composition_scores() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.composition_scores"]], "download_file_from_hf() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.download_file_from_hf"]], "gelu_fast() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.gelu_fast"]], "gelu_new() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.gelu_new"]], "get_act_name() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_act_name"]], "get_attention_mask() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_attention_mask"]], "get_corner() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_corner"]], "get_cumsum_along_dim() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_cumsum_along_dim"]], "get_dataset() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_dataset"]], "get_device() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_device"]], "get_input_with_manually_prepended_bos() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_input_with_manually_prepended_bos"]], "get_nested_attr() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_nested_attr"]], "get_offset_position_ids() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_offset_position_ids"]], "get_tokenizer_with_bos() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_tokenizer_with_bos"]], "get_tokens_with_bos_removed() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_tokens_with_bos_removed"]], "indices() (transformer_lens.utils.slice method)": [[69, "transformer_lens.utils.Slice.indices"]], "init_kaiming_normal_() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.init_kaiming_normal_"]], "init_kaiming_uniform_() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.init_kaiming_uniform_"]], "init_xavier_normal_() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.init_xavier_normal_"]], "init_xavier_uniform_() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.init_xavier_uniform_"]], "is_lower_triangular() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.is_lower_triangular"]], "is_square() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.is_square"]], "keep_single_column() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.keep_single_column"]], "lm_accuracy() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.lm_accuracy"]], "lm_cross_entropy_loss() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.lm_cross_entropy_loss"]], "override_or_use_default_value() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.override_or_use_default_value"]], "print_gpu_mem() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.print_gpu_mem"]], "remove_batch_dim() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.remove_batch_dim"]], "repeat_along_head_dimension() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.repeat_along_head_dimension"]], "sample_logits() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.sample_logits"]], "set_nested_attr() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.set_nested_attr"]], "slice (transformer_lens.utils.slice attribute)": [[69, "transformer_lens.utils.Slice.slice"]], "solu() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.solu"]], "test_prompt() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.test_prompt"]], "to_numpy() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.to_numpy"]], "tokenize_and_concatenate() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.tokenize_and_concatenate"]], "transformer_lens.utils": [[69, "module-transformer_lens.utils"]], "transpose() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.transpose"]], "unwrap() (transformer_lens.utils.slice class method)": [[69, "transformer_lens.utils.Slice.unwrap"]]}})
\ No newline at end of file
+Search.setIndex({"docnames": ["content/citation", "content/contributing", "content/gallery", "content/getting_started", "content/getting_started_mech_interp", "content/news/release-2.0", "content/special_cases", "content/tutorials", "generated/code/modules", "generated/code/transformer_lens", "generated/code/transformer_lens.ActivationCache", "generated/code/transformer_lens.FactoredMatrix", "generated/code/transformer_lens.HookedEncoder", "generated/code/transformer_lens.HookedEncoderDecoder", "generated/code/transformer_lens.HookedTransformer", "generated/code/transformer_lens.HookedTransformerConfig", "generated/code/transformer_lens.SVDInterpreter", "generated/code/transformer_lens.components", "generated/code/transformer_lens.components.abstract_attention", "generated/code/transformer_lens.components.attention", "generated/code/transformer_lens.components.bert_block", "generated/code/transformer_lens.components.bert_embed", "generated/code/transformer_lens.components.bert_mlm_head", "generated/code/transformer_lens.components.embed", "generated/code/transformer_lens.components.grouped_query_attention", "generated/code/transformer_lens.components.layer_norm", "generated/code/transformer_lens.components.layer_norm_pre", "generated/code/transformer_lens.components.pos_embed", "generated/code/transformer_lens.components.rms_norm", "generated/code/transformer_lens.components.rms_norm_pre", "generated/code/transformer_lens.components.t5_attention", "generated/code/transformer_lens.components.t5_block", "generated/code/transformer_lens.components.token_typed_embed", "generated/code/transformer_lens.components.transformer_block", "generated/code/transformer_lens.components.unembed", "generated/code/transformer_lens.evals", "generated/code/transformer_lens.head_detector", "generated/code/transformer_lens.hook_points", "generated/code/transformer_lens.loading_from_pretrained", "generated/code/transformer_lens.past_key_value_caching", "generated/code/transformer_lens.patching", "generated/code/transformer_lens.pretrained", "generated/code/transformer_lens.pretrained.weight_conversions", "generated/code/transformer_lens.pretrained.weight_conversions.bert", "generated/code/transformer_lens.pretrained.weight_conversions.bloom", "generated/code/transformer_lens.pretrained.weight_conversions.coder", "generated/code/transformer_lens.pretrained.weight_conversions.gemma", "generated/code/transformer_lens.pretrained.weight_conversions.gpt2", "generated/code/transformer_lens.pretrained.weight_conversions.gptj", "generated/code/transformer_lens.pretrained.weight_conversions.llama", "generated/code/transformer_lens.pretrained.weight_conversions.mingpt", "generated/code/transformer_lens.pretrained.weight_conversions.mistral", "generated/code/transformer_lens.pretrained.weight_conversions.mixtral", "generated/code/transformer_lens.pretrained.weight_conversions.nanogpt", "generated/code/transformer_lens.pretrained.weight_conversions.neel_solu_old", "generated/code/transformer_lens.pretrained.weight_conversions.neo", "generated/code/transformer_lens.pretrained.weight_conversions.neox", "generated/code/transformer_lens.pretrained.weight_conversions.opt", "generated/code/transformer_lens.pretrained.weight_conversions.phi", "generated/code/transformer_lens.pretrained.weight_conversions.phi3", "generated/code/transformer_lens.pretrained.weight_conversions.qwen", "generated/code/transformer_lens.pretrained.weight_conversions.qwen2", "generated/code/transformer_lens.pretrained.weight_conversions.t5", "generated/code/transformer_lens.train", "generated/code/transformer_lens.utilities", "generated/code/transformer_lens.utilities.activation_functions", "generated/code/transformer_lens.utilities.addmm", "generated/code/transformer_lens.utilities.attention", "generated/code/transformer_lens.utilities.devices", "generated/code/transformer_lens.utils", "generated/demos/Exploratory_Analysis_Demo", "generated/demos/Main_Demo", "generated/model_properties_table", "index"], "filenames": ["content/citation.md", "content/contributing.md", "content/gallery.md", "content/getting_started.md", "content/getting_started_mech_interp.md", "content/news/release-2.0.md", "content/special_cases.md", "content/tutorials.md", "generated/code/modules.rst", "generated/code/transformer_lens.rst", "generated/code/transformer_lens.ActivationCache.rst", "generated/code/transformer_lens.FactoredMatrix.rst", "generated/code/transformer_lens.HookedEncoder.rst", "generated/code/transformer_lens.HookedEncoderDecoder.rst", "generated/code/transformer_lens.HookedTransformer.rst", "generated/code/transformer_lens.HookedTransformerConfig.rst", "generated/code/transformer_lens.SVDInterpreter.rst", "generated/code/transformer_lens.components.rst", "generated/code/transformer_lens.components.abstract_attention.rst", "generated/code/transformer_lens.components.attention.rst", "generated/code/transformer_lens.components.bert_block.rst", "generated/code/transformer_lens.components.bert_embed.rst", "generated/code/transformer_lens.components.bert_mlm_head.rst", "generated/code/transformer_lens.components.embed.rst", "generated/code/transformer_lens.components.grouped_query_attention.rst", "generated/code/transformer_lens.components.layer_norm.rst", "generated/code/transformer_lens.components.layer_norm_pre.rst", "generated/code/transformer_lens.components.pos_embed.rst", "generated/code/transformer_lens.components.rms_norm.rst", "generated/code/transformer_lens.components.rms_norm_pre.rst", "generated/code/transformer_lens.components.t5_attention.rst", "generated/code/transformer_lens.components.t5_block.rst", "generated/code/transformer_lens.components.token_typed_embed.rst", "generated/code/transformer_lens.components.transformer_block.rst", "generated/code/transformer_lens.components.unembed.rst", "generated/code/transformer_lens.evals.rst", "generated/code/transformer_lens.head_detector.rst", "generated/code/transformer_lens.hook_points.rst", "generated/code/transformer_lens.loading_from_pretrained.rst", "generated/code/transformer_lens.past_key_value_caching.rst", "generated/code/transformer_lens.patching.rst", "generated/code/transformer_lens.pretrained.rst", "generated/code/transformer_lens.pretrained.weight_conversions.rst", "generated/code/transformer_lens.pretrained.weight_conversions.bert.rst", "generated/code/transformer_lens.pretrained.weight_conversions.bloom.rst", "generated/code/transformer_lens.pretrained.weight_conversions.coder.rst", "generated/code/transformer_lens.pretrained.weight_conversions.gemma.rst", "generated/code/transformer_lens.pretrained.weight_conversions.gpt2.rst", "generated/code/transformer_lens.pretrained.weight_conversions.gptj.rst", "generated/code/transformer_lens.pretrained.weight_conversions.llama.rst", "generated/code/transformer_lens.pretrained.weight_conversions.mingpt.rst", "generated/code/transformer_lens.pretrained.weight_conversions.mistral.rst", "generated/code/transformer_lens.pretrained.weight_conversions.mixtral.rst", "generated/code/transformer_lens.pretrained.weight_conversions.nanogpt.rst", "generated/code/transformer_lens.pretrained.weight_conversions.neel_solu_old.rst", "generated/code/transformer_lens.pretrained.weight_conversions.neo.rst", "generated/code/transformer_lens.pretrained.weight_conversions.neox.rst", "generated/code/transformer_lens.pretrained.weight_conversions.opt.rst", "generated/code/transformer_lens.pretrained.weight_conversions.phi.rst", "generated/code/transformer_lens.pretrained.weight_conversions.phi3.rst", "generated/code/transformer_lens.pretrained.weight_conversions.qwen.rst", "generated/code/transformer_lens.pretrained.weight_conversions.qwen2.rst", "generated/code/transformer_lens.pretrained.weight_conversions.t5.rst", "generated/code/transformer_lens.train.rst", "generated/code/transformer_lens.utilities.rst", "generated/code/transformer_lens.utilities.activation_functions.rst", "generated/code/transformer_lens.utilities.addmm.rst", "generated/code/transformer_lens.utilities.attention.rst", "generated/code/transformer_lens.utilities.devices.rst", "generated/code/transformer_lens.utils.rst", "generated/demos/Exploratory_Analysis_Demo.ipynb", "generated/demos/Main_Demo.ipynb", "generated/model_properties_table.md", "index.md"], "titles": ["Citation", "Contributing", "Gallery", "Getting Started", "Getting Started in Mechanistic Interpretability", "TransformerLens 2.0", "Special Cases", "Tutorials", "Transformer Lens API", "transformer_lens", "transformer_lens.ActivationCache", "transformer_lens.FactoredMatrix", "transformer_lens.HookedEncoder", "transformer_lens.HookedEncoderDecoder", "transformer_lens.HookedTransformer", "transformer_lens.HookedTransformerConfig", "transformer_lens.SVDInterpreter", "transformer_lens.components", "transformer_lens.components.abstract_attention", "transformer_lens.components.attention", "transformer_lens.components.bert_block", "transformer_lens.components.bert_embed", "transformer_lens.components.bert_mlm_head", "transformer_lens.components.embed", "transformer_lens.components.grouped_query_attention", "transformer_lens.components.layer_norm", "transformer_lens.components.layer_norm_pre", "transformer_lens.components.pos_embed", "transformer_lens.components.rms_norm", "transformer_lens.components.rms_norm_pre", "transformer_lens.components.t5_attention", "transformer_lens.components.t5_block", "transformer_lens.components.token_typed_embed", "transformer_lens.components.transformer_block", "transformer_lens.components.unembed", "transformer_lens.evals", "transformer_lens.head_detector", "transformer_lens.hook_points", "transformer_lens.loading_from_pretrained", "transformer_lens.past_key_value_caching", "transformer_lens.patching", "transformer_lens.pretrained", "transformer_lens.pretrained.weight_conversions", "transformer_lens.pretrained.weight_conversions.bert", "transformer_lens.pretrained.weight_conversions.bloom", "transformer_lens.pretrained.weight_conversions.coder", "transformer_lens.pretrained.weight_conversions.gemma", "transformer_lens.pretrained.weight_conversions.gpt2", "transformer_lens.pretrained.weight_conversions.gptj", "transformer_lens.pretrained.weight_conversions.llama", "transformer_lens.pretrained.weight_conversions.mingpt", "transformer_lens.pretrained.weight_conversions.mistral", "transformer_lens.pretrained.weight_conversions.mixtral", "transformer_lens.pretrained.weight_conversions.nanogpt", "transformer_lens.pretrained.weight_conversions.neel_solu_old", "transformer_lens.pretrained.weight_conversions.neo", "transformer_lens.pretrained.weight_conversions.neox", "transformer_lens.pretrained.weight_conversions.opt", "transformer_lens.pretrained.weight_conversions.phi", "transformer_lens.pretrained.weight_conversions.phi3", "transformer_lens.pretrained.weight_conversions.qwen", "transformer_lens.pretrained.weight_conversions.qwen2", "transformer_lens.pretrained.weight_conversions.t5", "transformer_lens.train", "transformer_lens.utilities", "transformer_lens.utilities.activation_functions", "transformer_lens.utilities.addmm", "transformer_lens.utilities.attention", "transformer_lens.utilities.devices", "transformer_lens.utils", "Exploratory Analysis Demo", "Transformer Lens Main Demo Notebook", "Model Properties Table", "TransformerLens"], "terms": {"pleas": [0, 1, 3, 4, 5, 71], "cite": 0, "thi": [0, 1, 3, 4, 5, 6, 7, 10, 11, 12, 13, 14, 15, 16, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 32, 33, 34, 35, 36, 37, 38, 39, 40, 63, 67, 68, 69, 71, 73], "librari": [0, 2, 3, 4, 5, 7, 10, 35, 69, 70], "misc": 0, "nanda2022transformerlen": 0, "titl": [0, 1, 70, 71], "transformerlen": [0, 2, 3, 4, 7, 10, 14, 18, 38, 69, 70, 71], "author": [0, 70], "neel": [0, 2, 4, 7, 14, 16, 71], "nanda": [0, 2, 4, 14, 71], "joseph": [0, 5], "bloom": [0, 5, 18, 38, 42, 72], "year": 0, "2022": [0, 69], "howpublish": 0, "url": [0, 3], "http": [0, 1, 3, 7, 10, 14, 15, 18, 24, 32, 35, 36, 40, 66, 69, 70, 71], "github": [0, 1, 3, 7, 14, 66], "com": [0, 3, 7, 10, 14, 18, 66, 70, 71], "transformerlensorg": [0, 3, 7], "For": [1, 5, 10, 12, 13, 14, 18, 32, 36, 69, 70], "one": [1, 3, 4, 5, 10, 12, 13, 14, 15, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 36, 37, 38, 39, 40, 69, 70, 71, 73], "click": [1, 71], "your": [1, 3, 5, 7, 14, 15, 36, 37, 70, 71], "develop": [1, 5, 7, 70, 71], "environ": [1, 3], "project": [1, 5, 7, 10, 18, 24, 63, 70], "includ": [1, 4, 5, 7, 10, 12, 13, 14, 15, 35, 36, 37, 70], "It": [1, 3, 5, 7, 10, 12, 13, 14, 15, 18, 35, 37, 40, 69, 70, 71, 73], "can": [1, 2, 3, 5, 6, 7, 10, 11, 12, 13, 14, 15, 16, 18, 19, 24, 35, 36, 37, 38, 39, 40, 69, 70, 71, 73], "us": [1, 2, 3, 5, 6, 7, 10, 12, 13, 14, 15, 16, 18, 19, 24, 26, 31, 33, 35, 36, 37, 38, 39, 40, 63, 67, 68, 69, 70, 71, 73], "local": [1, 14, 15, 18, 19, 24, 38, 69, 71], "v": [1, 10, 12, 13, 14, 15, 18, 24, 40, 70, 71], "code": [1, 4, 5, 10, 14, 15, 18, 35, 36, 37, 38, 69, 70, 71], "codespac": 1, "poetri": 1, "packag": 1, "manag": [1, 5, 10, 14, 37, 69], "instal": [1, 5, 70, 71], "follow": [1, 3, 5, 10, 14, 69, 71, 73], "also": [1, 5, 7, 10, 12, 13, 14, 15, 16, 31, 36, 37, 38, 68, 69, 70, 71], "virtual": 1, "config": [1, 14, 15, 18, 19, 24, 38, 40, 63], "virtualenv": 1, "true": [1, 10, 12, 13, 14, 15, 30, 31, 35, 36, 37, 38, 40, 68, 69, 70, 71], "dev": 1, "doc": [1, 5, 8, 10, 14, 71], "jupyt": 1, "If": [1, 3, 5, 8, 10, 12, 13, 14, 15, 25, 28, 32, 36, 37, 38, 40, 68, 69, 70, 71], "ad": [1, 7, 14, 15, 18, 37, 70, 71], "featur": [1, 3, 5, 7, 12, 13, 16, 18, 40, 69, 70, 73], "add": [1, 5, 14, 15, 18, 37, 39, 66, 69, 70, 71, 73], "unit": [1, 5], "you": [1, 3, 4, 5, 7, 10, 12, 13, 14, 15, 16, 30, 35, 36, 37, 38, 69, 70, 71, 73], "need": [1, 3, 5, 10, 14, 15, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 69, 70, 71, 73], "model": [1, 2, 3, 6, 7, 10, 12, 13, 14, 15, 16, 18, 19, 21, 24, 35, 36, 37, 38, 40, 45, 63, 68, 69, 70], "ones": [1, 12, 13, 14, 31, 36, 70], "ar": [1, 3, 4, 5, 6, 7, 10, 11, 12, 13, 14, 15, 18, 20, 24, 35, 36, 37, 38, 39, 40, 69, 71, 73], "cach": [1, 10, 14, 15, 31, 33, 36, 37, 39, 40, 69, 70, 73], "action": [1, 2, 70], "so": [1, 5, 7, 10, 11, 12, 13, 14, 15, 16, 35, 37, 38, 39, 40, 69, 70, 71], "quickli": [1, 5, 7], "cd": [1, 70, 71], "These": [1, 70, 71], "gpt2": [1, 14, 15, 16, 18, 19, 24, 35, 38, 42, 70, 71, 72], "attn": [1, 10, 12, 13, 14, 15, 18, 30, 38, 40, 69, 70, 71, 72], "onli": [1, 2, 5, 10, 11, 12, 13, 14, 15, 18, 19, 24, 26, 31, 33, 36, 37, 38, 69, 70, 71, 72], "1l": [1, 38, 70, 71, 72], "2l": [1, 14, 38, 71, 72], "3l": [1, 38, 71, 72], "4l": [1, 38, 71, 72], "tini": [1, 10, 14, 38, 69, 70, 71, 72], "stori": [1, 10, 14, 38, 40, 69, 70, 72], "1m": [1, 10, 14, 38, 69, 72], "note": [1, 3, 5, 10, 11, 12, 13, 14, 15, 18, 24, 35, 37, 38, 66, 69, 70, 71], "i": [1, 3, 4, 5, 6, 7, 10, 11, 12, 13, 14, 15, 16, 18, 22, 24, 26, 31, 32, 33, 35, 36, 37, 38, 39, 40, 63, 67, 68, 69, 70, 73], "quit": [1, 5], "slow": [1, 71], "we": [1, 2, 5, 8, 10, 14, 15, 36, 39, 40, 69, 70, 71, 73], "have": [1, 3, 5, 10, 12, 13, 14, 18, 19, 24, 36, 40, 69, 70, 71, 73], "cpu": [1, 10, 12, 13, 14, 15, 18, 38, 70, 71], "smaller": [1, 5, 71], "like": [1, 3, 4, 5, 7, 12, 13, 14, 15, 21, 30, 35, 36, 40, 69, 70, 71, 73], "prefer": 1, "possibl": [1, 5, 12, 13, 14, 36, 40, 69, 70, 71, 73], "via": [1, 2, 3, 4, 5, 12, 13, 14, 40, 70], "make": [1, 3, 5, 7, 11, 12, 13, 14, 36, 37, 70, 71, 73], "accept": [1, 3, 5, 12, 13, 14, 37, 70], "notebook": [1, 3, 7, 70, 73], "all": [1, 4, 5, 10, 12, 13, 14, 15, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 32, 33, 34, 35, 36, 37, 40, 65, 69, 70], "suit": 1, "mention": [1, 5, 71], "pycln": 1, "isort": 1, "black": [1, 71], "pull": [1, 5], "request": [1, 5], "check": [1, 3, 5, 7, 14, 16, 18, 35, 36, 37, 69, 70, 71], "file": [1, 5, 69], "line": [1, 5, 70, 71], "length": [1, 10, 12, 13, 14, 15, 18, 25, 26, 27, 28, 29, 69, 70, 71], "set": [1, 2, 5, 10, 12, 13, 14, 15, 18, 35, 36, 37, 40, 63, 69, 70, 71], "100": [1, 35, 70, 71], "pyproject": 1, "toml": 1, "instead": [1, 6, 10, 14, 15, 18, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 70, 71], "default": [1, 6, 10, 14, 15, 16, 18, 19, 24, 27, 31, 33, 35, 36, 37, 38, 40, 69, 70, 71], "88": [1, 71], "sure": [1, 3, 5, 14, 69, 70, 71], "thorough": 1, "ani": [1, 3, 10, 12, 13, 14, 15, 18, 19, 24, 37, 69, 70, 71, 73], "should": [1, 5, 7, 10, 12, 13, 14, 18, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 36, 37, 63, 69, 70, 71], "do": [1, 3, 4, 5, 7, 10, 12, 13, 14, 18, 35, 37, 40, 69, 70, 71, 73], "directli": [1, 5, 12, 13, 15, 69, 70, 71], "automat": [1, 5, 7, 14, 15, 69, 70, 71], "gener": [1, 5, 7, 13, 14, 18, 31, 33, 35, 39, 40, 69, 70], "api": [1, 5, 37, 70], "when": [1, 3, 5, 7, 10, 11, 12, 13, 14, 15, 24, 31, 33, 35, 37, 38, 39, 40, 67, 69, 70, 71], "merg": [1, 5, 14], "main": [1, 3, 5, 6, 7, 10, 37, 70], "thei": [1, 4, 5, 14, 15, 18, 35, 40, 69, 70, 71, 73], "pytest": 1, "doctest": 1, "want": [1, 5, 7, 10, 14, 16, 35, 36, 37, 39, 69, 70, 71], "view": [1, 2], "chang": [1, 2, 3, 5, 14, 15, 37, 38, 40, 69, 70, 71], "hot": [1, 70, 71], "reload": [1, 70, 71], "give": [1, 5, 10, 14, 15, 35, 38, 40, 69, 70, 71], "real": [1, 7, 69, 70, 71, 73], "time": [1, 5, 7, 8, 10, 14, 36, 37, 69, 70, 71], "edit": [1, 7, 14, 40, 70, 71, 73], "googl": [1, 7, 38, 70, 71], "python": [1, 2, 15, 30, 35, 38, 69, 71, 72], "write": [1, 2, 3, 5, 14, 69, 70, 71, 73], "some": [1, 3, 5, 10, 14, 16, 18, 35, 37, 40, 69, 70], "from": [1, 2, 3, 4, 5, 6, 7, 10, 11, 12, 13, 14, 15, 16, 18, 24, 32, 35, 36, 38, 40, 69, 70, 71, 73], "restructuredtext": 1, "rest": [1, 5, 10, 14, 15, 35, 38, 69, 71], "In": [1, 2, 5, 6, 10, 12, 13, 14, 35, 37, 70, 71], "case": [1, 2, 5, 10, 14, 15, 35, 36, 37, 38, 40, 69, 70, 72], "A": [1, 2, 4, 5, 10, 11, 12, 13, 14, 18, 24, 31, 32, 33, 35, 37, 39, 40, 69, 70, 71], "descript": 1, "what": [1, 3, 5, 7, 10, 14, 36, 40, 71, 73], "doe": [1, 5, 10, 12, 13, 14, 15, 36, 37, 40, 69, 70, 71], "much": [1, 5, 10, 14, 35, 36, 40, 69, 70, 71], "detail": [1, 5, 10, 14, 15, 18, 24, 33, 38, 40, 69, 70, 71], "necessari": [1, 5, 71], "fulli": [1, 40, 70], "understand": [1, 10, 14, 36, 71], "warn": [1, 10, 14, 36, 37, 69], "user": [1, 2, 5, 15, 69, 71], "e": [1, 5, 10, 12, 13, 14, 15, 18, 36, 37, 38, 69, 70, 71, 72], "g": [1, 5, 10, 12, 13, 14, 36, 37, 69, 71], "common": [1, 5, 7, 10, 14, 15, 18, 69, 70, 71], "pitfal": 1, "exampl": [1, 2, 10, 12, 13, 14, 16, 18, 32, 35, 37, 69, 70], "here": [1, 2, 3, 5, 14, 15, 18, 19, 24, 35, 36, 69, 70, 71], "print": [1, 10, 35, 63, 69, 70, 71], "1": [1, 3, 4, 5, 10, 12, 13, 14, 15, 18, 19, 20, 24, 30, 31, 32, 36, 37, 38, 39, 69, 70, 71, 72], "2": [1, 3, 4, 10, 12, 13, 14, 15, 18, 35, 36, 38, 69, 70, 71, 72, 73], "3": [1, 6, 10, 11, 12, 13, 14, 18, 35, 38, 40, 69, 70, 71, 72, 73], "arg": [1, 24, 37], "param_without_type_signatur": 1, "each": [1, 5, 10, 11, 12, 13, 14, 15, 18, 36, 37, 38, 39, 40, 67, 69, 70, 71], "indent": 1, "onc": [1, 3, 5, 14, 69, 70, 71], "more": [1, 5, 7, 10, 11, 14, 15, 18, 32, 36, 40, 69, 70, 71, 73], "param_2": 1, "anoth": [1, 5, 70, 71], "paramet": [1, 5, 7, 10, 12, 13, 14, 15, 16, 18, 19, 24, 25, 26, 27, 31, 33, 35, 36, 37, 38, 40, 68, 69, 70], "return": [1, 5, 10, 11, 12, 13, 14, 18, 24, 27, 31, 33, 35, 36, 37, 38, 40, 63, 68, 69, 70, 71], "without": [1, 3, 5, 10, 14, 28, 29, 69, 70, 71], "type": [1, 6, 7, 10, 12, 13, 14, 15, 16, 21, 24, 27, 31, 32, 33, 36, 37, 38, 40, 63, 68, 69, 70, 71], "signatur": [1, 12, 13, 14, 71], "rais": [1, 14, 36, 38, 69, 71], "inform": [1, 14, 15, 32, 37, 38, 70, 71], "about": [1, 5, 7, 10, 14, 35, 37, 40, 69, 70, 71, 73], "error": [1, 10, 14, 36, 38, 71], "mai": [1, 5, 10, 12, 13, 14, 15, 18, 69, 70, 71], "part": [1, 5, 10, 14, 26, 40, 70, 71, 73], "codebas": [1, 71], "cross": [1, 10, 14, 30, 31, 69, 70, 71], "referenc": [1, 5], "omit": [1, 36, 71], "full": [1, 4, 5, 10, 12, 14, 15, 18, 69, 71], "path": [1, 4, 69], "same": [1, 3, 5, 10, 11, 14, 15, 18, 36, 37, 39, 40, 67, 69, 70, 71], "mod": 1, "transformer_len": [1, 3, 5, 8, 70, 71], "modul": [1, 5, 8, 12, 13, 14, 15, 16, 18, 19, 20, 21, 22, 23, 25, 26, 27, 28, 29, 31, 32, 33, 34, 35, 37, 38, 39, 40, 69, 71], "const": 1, "loading_from_pretrain": [1, 8, 9, 14, 71], "official_model_nam": [1, 14, 38], "hookedtransform": [1, 3, 5, 6, 7, 8, 9, 10, 12, 13, 15, 16, 35, 36, 38, 40, 63, 68, 69, 70, 71], "meth": [1, 10], "from_pretrain": [1, 3, 6, 10, 12, 13, 14, 16, 35, 38, 69, 70, 71], "attr": 1, "cfg": [1, 12, 13, 14, 16, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 39, 44, 45, 51, 52, 58, 59, 60, 61, 62, 68, 69, 70, 71], "latex": 1, "re": [1, 3, 4, 7, 10, 14, 15, 36, 70, 71], "place": [1, 5, 7, 12, 13, 14, 69, 70, 71], "string": [1, 12, 13, 14, 15, 35, 36, 37, 38, 69, 70, 71], "backward": [1, 37, 70, 71], "slash": 1, "must": [1, 10, 12, 13, 14, 15, 18, 36, 37, 66, 69, 70, 71], "repeat": [1, 14, 35, 39, 69, 70, 71], "inlin": 1, "displai": [1, 70, 71], "mode": [1, 10, 14, 15, 18, 26, 69, 70, 71], "b": [1, 11, 12, 14, 32, 35, 38, 40, 67, 69, 71, 72], "2ab": 1, "nowrap": 1, "begin": [1, 3, 5, 13, 14, 35, 36, 69, 70, 71], "eqnarrai": 1, "y": [1, 10, 14, 36, 70, 71], "ax": [1, 14, 40, 71], "bx": 1, "c": [1, 38, 69, 71, 72], "f": [1, 70, 71], "x": [1, 5, 10, 14, 18, 25, 26, 28, 29, 36, 37, 38, 66, 69, 70, 71], "2xy": 1, "end": [1, 5, 7, 14, 37, 40, 69, 70, 71], "ital": 1, "text": [1, 7, 10, 14, 15, 18, 31, 33, 35, 37, 38, 39, 69, 70], "bold": 1, "list": [1, 3, 4, 10, 12, 13, 14, 15, 35, 36, 37, 39, 40, 69, 70, 71], "item": [1, 5, 10, 69, 70, 71], "number": [1, 5, 10, 14, 15, 16, 18, 35, 38, 40, 63, 68, 69, 70, 71], "quot": 1, "level": [1, 37, 70, 71, 73], "extern": [1, 70], "link": [1, 14, 35], "domain": 1, "invalid": 1, "research": [2, 3, 4, 5, 7, 70, 71, 73], "done": [2, 4, 5, 6, 10, 14, 15, 18, 37, 70, 71], "involv": [2, 5, 70, 71], "progress": [2, 5, 14, 71], "measur": [2, 35, 36, 40, 69, 70], "grokk": [2, 7], "mechanist": [2, 3, 7, 40, 70, 71], "interpret": [2, 3, 7, 10, 14, 16, 36, 40, 69, 70], "iclr": 2, "spotlight": 2, "2023": 2, "lawrenc": 2, "chan": 2, "tom": [2, 70], "lieberum": 2, "jess": 2, "smith": 2, "jacob": 2, "steinhardt": 2, "find": [2, 5, 7, 10, 11, 14, 40, 70, 71], "neuron": [2, 7, 10, 14, 70, 71], "haystack": 2, "studi": [2, 4, 40, 70, 71], "spars": [2, 5], "probe": 2, "gurne": 2, "matthew": 2, "pauli": 2, "katherin": 2, "harvei": 2, "dmitrii": 2, "troitskii": 2, "dimitri": 2, "bertsima": 2, "toward": [2, 18, 40, 70], "autom": 2, "circuit": [2, 12, 14, 18, 35, 36, 40, 69, 70, 71], "discoveri": 2, "arthur": [2, 71], "conmi": [2, 71], "augustin": 2, "n": [2, 14, 18, 63, 66, 69, 70, 71], "mavor": 2, "parker": 2, "aengu": 2, "lynch": 2, "stefan": 2, "heimersheim": 2, "adri\u00e0": 2, "garriga": 2, "alonso": 2, "actual": [2, 5, 14, 36, 37, 71], "othello": [2, 7, 38, 72], "gpt": [2, 3, 4, 7, 10, 12, 13, 14, 15, 18, 19, 24, 35, 38, 69, 70, 71, 72, 73], "ha": [2, 3, 4, 5, 10, 11, 12, 13, 14, 18, 24, 35, 38, 39, 40, 67, 69, 70, 71], "linear": [2, 7, 12, 13, 14, 18, 38, 67, 70, 71], "emerg": [2, 7], "world": [2, 5, 7, 71, 73], "represent": [2, 7], "docstr": 2, "4": [2, 3, 5, 15, 18, 35, 69, 70, 71, 72], "layer": [2, 6, 10, 12, 13, 14, 15, 16, 18, 19, 21, 24, 25, 26, 36, 37, 38, 39, 40, 67, 68, 69, 71], "attent": [2, 7, 10, 12, 13, 14, 15, 17, 18, 20, 24, 27, 30, 31, 33, 36, 40, 64, 69, 71], "transform": [2, 3, 4, 7, 10, 12, 13, 14, 15, 18, 19, 20, 21, 22, 23, 25, 26, 27, 28, 29, 31, 32, 33, 34, 38, 39, 40, 66, 69, 70], "jett": 2, "janiak": 2, "toi": [2, 14], "univers": 2, "icml": 2, "bilal": 2, "chughtai": 2, "n2g": 2, "scalabl": 2, "approach": [2, 5, 10, 70, 71], "quantifi": [2, 36], "larg": [2, 5, 7, 15, 18, 38, 69, 71, 72, 73], "languag": [2, 12, 14, 35, 63, 69, 70, 71], "workshop": 2, "rtml": 2, "alex": [2, 71], "foot": [2, 14, 71], "esben": 2, "kran": 2, "ioanni": 2, "konsta": 2, "fazl": 2, "barez": 2, "elicit": 2, "latent": 2, "predict": [2, 7, 10, 12, 14, 22, 35, 36, 69, 70, 71], "tune": [2, 12, 13, 38, 69, 71, 72], "len": [2, 10, 38], "nora": 2, "belros": 2, "zach": 2, "furman": 2, "logan": 2, "danni": 2, "halawi": 2, "igor": 2, "ostrovski": 2, "lev": 2, "mckinnei": 2, "stella": 2, "biderman": 2, "contribut": [2, 5, 10, 14, 70], "being": [2, 5, 10, 12, 13, 14, 15, 36, 37, 40, 69, 70, 71], "induct": [2, 4, 35, 36, 38], "head": [2, 4, 7, 10, 12, 13, 14, 15, 16, 18, 22, 24, 35, 36, 38, 40, 67, 69], "phase": 2, "replic": [2, 4, 14, 16, 35, 70, 71], "partial": [2, 70, 71], "context": [2, 10, 14, 37, 40, 69, 70, 71], "learn": [2, 3, 7, 15, 63, 69, 70, 71, 73], "connor": 2, "kissan": 2, "decis": [2, 3], "script": [2, 7], "train": [2, 7, 8, 9, 10, 12, 13, 14, 15, 35, 38, 69, 70, 73], "which": [2, 3, 5, 6, 7, 10, 12, 13, 14, 15, 35, 36, 37, 38, 39, 40, 69, 70, 71], "intermedi": [2, 10, 14, 37, 71], "activ": [2, 3, 4, 5, 7, 10, 12, 13, 14, 15, 16, 37, 40, 65, 69, 73], "perform": [2, 6, 7, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 35, 37, 69, 70, 71], "attribut": [2, 4, 7, 10, 18, 40, 69, 71], "ablat": [2, 70, 71], "up": [2, 3, 4, 5, 10, 14, 15, 37, 40, 63, 69, 70, 71], "initi": [2, 5, 8, 14, 15, 24, 37, 68, 69, 70, 71], "work": [2, 3, 4, 5, 7, 10, 12, 13, 14, 18, 37, 38, 69, 70, 71, 73], "found": [2, 3, 5, 6, 14, 15, 70, 71], "demo": [3, 5, 16, 38, 72], "how": [3, 5, 7, 10, 14, 36, 40, 63, 70, 71, 73], "basic": [3, 7, 14, 35, 69, 70], "To": [3, 4, 5, 10, 14, 15, 18, 37, 38, 70, 71], "see": [3, 5, 7, 10, 12, 13, 14, 15, 18, 24, 32, 33, 36, 38, 40, 69, 70, 71, 73], "exploratori": [3, 7, 36, 69, 71, 73], "analysi": [3, 7, 10, 14, 36, 69, 71, 73], "practic": [3, 4, 7, 70, 71], "look": [3, 4, 5, 7, 8, 10, 14, 18, 36, 40, 70, 71], "out": [3, 5, 7, 10, 14, 16, 40, 69, 70, 71], "my": [3, 5, 7, 14, 15, 69, 70, 71, 73], "analys": [3, 7, 10, 14, 71], "indirect": [3, 4, 7, 35], "object": [3, 4, 5, 7, 10, 11, 12, 13, 14, 15, 16, 35, 37, 38, 39, 63, 69], "identif": [3, 4, 7, 35], "record": [3, 7, 71], "myself": [3, 5, 7, 71], "veri": [3, 4, 5, 7, 10, 15, 16, 35, 69, 70, 71, 73], "young": [3, 4, 71], "small": [3, 4, 5, 6, 7, 10, 14, 15, 35, 38, 69, 70, 71, 72, 73], "field": [3, 4, 5, 14, 69, 71, 73], "lot": [3, 4, 5, 7, 10, 11, 39, 40, 69, 70, 71, 73], "open": [3, 4, 5, 14, 35, 73], "problem": [3, 4, 5, 71, 73], "would": [3, 4, 5, 12, 18, 32, 70, 71, 73], "help": [3, 4, 5, 15, 40, 70, 71], "try": [3, 4, 10, 14, 36, 70, 71], "concret": [3, 4, 70, 71], "figur": [3, 40, 70, 71], "where": [3, 5, 6, 10, 11, 12, 13, 14, 15, 18, 33, 36, 37, 38, 40, 63, 69, 70, 71], "skill": [3, 71], "kei": [3, 4, 5, 10, 12, 13, 14, 15, 18, 19, 24, 30, 31, 33, 35, 36, 39, 40, 69, 70, 71], "resourc": [3, 4, 5], "new": [3, 7, 10, 14, 37, 38, 39, 69, 70, 71], "tutori": [3, 4, 5, 70, 71], "scratch": [3, 4, 70], "an": [3, 4, 7, 10, 11, 12, 13, 14, 15, 18, 35, 36, 37, 38, 39, 40, 63, 67, 69, 70, 73], "accompani": [3, 4, 7, 71], "templat": [3, 35], "yourself": [3, 14, 70, 71], "One": [3, 5, 14, 70, 71, 73], "signific": [3, 37, 70, 71], "design": [3, 5, 10, 70, 71, 73], "made": [3, 5, 35, 70, 71], "wa": [3, 5, 6, 10, 12, 13, 14, 15, 35, 40, 70, 71], "singl": [3, 5, 10, 12, 14, 18, 31, 32, 33, 39, 40, 69, 70, 71], "implement": [3, 5, 12, 13, 14, 18, 40, 66, 69, 70, 71], "could": [3, 5, 70, 71], "support": [3, 5, 7, 12, 13, 14, 15, 30, 36, 37, 65, 66, 69, 70, 71], "rang": [3, 4, 5, 14, 16, 36, 40, 69, 70, 71], "subtli": [3, 18], "differ": [3, 5, 6, 10, 12, 13, 14, 15, 18, 35, 36, 37, 40, 69, 70, 71], "style": [3, 5, 10, 12, 13, 14, 15, 19, 36, 70, 71, 73], "upsid": 3, "just": [3, 4, 5, 10, 14, 15, 35, 40, 69, 70, 71], "arbitrari": [3, 14, 70, 71], "name": [3, 5, 10, 14, 15, 35, 36, 37, 38, 40, 63, 69], "But": [3, 10, 14, 40, 69, 70, 71], "downsid": 3, "py": [3, 5, 12, 13, 66], "compon": [3, 8, 9, 10, 12, 13, 14, 15, 67, 69, 70, 71], "difficult": [3, 10], "recommend": [3, 8, 10, 14, 15, 16, 37, 70, 71], "clean": [3, 40, 69, 70, 71], "minim": [3, 5, 71], "intern": [3, 5, 10, 14, 40, 70, 71, 73], "architectur": [3, 12, 13, 70], "significantli": [3, 12, 13, 14, 35, 40, 70, 71], "clearer": 3, "better": [3, 14, 15, 35, 36, 38, 70, 71], "document": [3, 14, 69, 71], "pip": [3, 5, 70, 71], "git": 3, "import": [3, 5, 10, 14, 16, 35, 39, 40, 69, 73], "known": [3, 73], "easytransform": [3, 71, 73], "break": [3, 5, 10, 70, 71], "been": [3, 5, 10, 14, 69, 71], "sinc": [3, 5, 10, 14, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 70, 71], "renam": [3, 5], "old": [3, 5, 38, 71], "version": [3, 7, 14, 35, 37, 66, 70, 71], "legaci": [3, 36], "run": [3, 5, 6, 10, 14, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 38, 39, 40, 63, 70, 73], "v1": 3, "avail": [3, 5, 7, 10, 14, 15, 36, 38], "requir": [3, 5, 12, 13, 14, 40, 69], "luckili": 3, "provid": [3, 10, 12, 13, 14, 25, 28, 32, 37, 68, 69], "wai": [3, 5, 6, 10, 14, 15, 37, 69, 70, 71], "those": [3, 5, 6, 14, 37, 69, 70], "configur": [3, 5, 15, 63, 68], "environment": 3, "variabl": [3, 12, 13], "simpli": [3, 5, 70], "token": [3, 7, 10, 12, 13, 14, 15, 16, 18, 19, 21, 22, 23, 24, 27, 31, 32, 33, 35, 36, 38, 40, 69, 70], "hf_token": 3, "agreement": 3, "issu": [3, 5, 14, 70, 71], "attempt": [3, 5, 14], "ue": 3, "befor": [3, 10, 12, 13, 14, 15, 18, 19, 20, 37, 69, 70, 71], "relat": [3, 14, 18, 70, 71], "consol": 3, "output": [3, 5, 7, 10, 12, 13, 14, 15, 18, 36, 37, 40, 69, 70, 71], "point": [3, 5, 10, 12, 13, 14, 15, 16, 33, 37, 69, 70, 73], "As": [3, 14, 15, 69, 70, 71], "23": [3, 70, 71], "24": [3, 14, 69, 70, 71, 72], "current": [3, 5, 10, 12, 13, 14, 15, 18, 19, 24, 36, 70, 71], "co": [3, 69], "mistralai": [3, 38], "mixtral": [3, 5, 6, 38, 42, 72], "8x7b": [3, 38], "v0": [3, 38, 72], "mistral": [3, 15, 18, 35, 38, 42, 72], "7b": [3, 5, 7, 38, 71, 72], "instruct": [3, 38, 71, 72], "mean": [4, 5, 10, 14, 15, 16, 18, 19, 24, 28, 29, 36, 37, 69, 70, 71], "": [4, 5, 7, 10, 11, 12, 13, 14, 15, 16, 18, 35, 36, 37, 38, 40, 68, 69, 71, 73], "both": [4, 5, 10, 14, 15, 18, 36, 37, 39, 66, 70, 71], "low": [4, 11, 14, 18, 69, 71], "hang": [4, 71], "fruit": [4, 71], "bar": [4, 14], "entri": [4, 18, 39, 40, 71], "The": [4, 5, 7, 10, 11, 12, 13, 14, 15, 16, 18, 19, 22, 24, 27, 31, 32, 33, 35, 36, 37, 38, 39, 40, 63, 68, 69, 70, 71, 73], "standard": [4, 6, 14, 15, 70, 71], "answer": [4, 10, 40, 69, 70, 71], "why": [4, 5, 10, 18, 69, 70, 71], "yet": [4, 5, 12, 13, 14, 70, 71, 73], "aren": [4, 38, 71], "t": [4, 5, 10, 11, 12, 13, 14, 15, 16, 18, 35, 38, 69, 70, 71, 73], "enough": [4, 5, 10, 70, 71], "peopl": [4, 5, 71], "guid": [4, 71], "arena": 4, "callum": [4, 71], "mcdougal": [4, 71], "comprehens": [4, 71], "introduct": 4, "mech": [4, 70], "interp": [4, 70], "written": [4, 5, 7, 70], "snippet": 4, "copi": [4, 12, 13, 70], "come": [4, 5, 14, 15, 40, 70, 71], "exercis": [4, 70], "solut": [4, 5, 70, 71], "notabl": [4, 14, 37, 70, 71], "video": [4, 7, 70, 71], "me": [4, 5, 38, 71, 73], "good": [4, 5, 7, 10, 35, 69, 70, 71, 73], "cover": [4, 14, 71], "foundat": [4, 71], "concept": [4, 70, 71], "wild": [4, 10, 70, 71], "techniqu": [4, 7, 14, 40, 70, 71], "direct": [4, 5, 10, 14, 16, 37, 40, 71], "logit": [4, 5, 6, 7, 10, 12, 13, 14, 15, 22, 35, 40, 69, 71], "patch": [4, 5, 7, 8, 9], "paper": [4, 7, 10, 14, 15, 18, 32, 35, 40, 71], "read": [4, 5, 7, 10, 14, 71], "200": [4, 71], "explain": [4, 7, 70, 71], "jargon": 4, "unfamiliar": [4, 70], "term": [4, 10, 14, 70], "go": [4, 5, 7, 40, 71], "across": [4, 5, 10, 12, 13, 14, 16, 38, 40, 68, 70, 71], "youtub": 4, "channel": 4, "content": [4, 35, 70, 71], "walkthrough": [4, 70, 71], "am": 5, "happi": 5, "announc": 5, "now": [5, 7, 14, 15, 70, 71], "releas": 5, "recent": 5, "primari": 5, "motiv": [5, 70], "behind": [5, 18, 70], "jump": [5, 71], "transit": [5, 70], "strictli": [5, 10, 71], "describ": [5, 15, 69, 70], "At": [5, 70], "last": [5, 10, 14, 69, 71], "minut": 5, "did": [5, 10, 69, 70, 71], "remov": [5, 10, 11, 14, 18, 37, 69, 70, 71, 73], "hookedsa": 5, "had": [5, 70, 71], "saelen": 5, "bundl": [5, 71], "major": 5, "hand": [5, 70, 71], "modif": 5, "affect": [5, 14, 40, 70], "bryce": 5, "meyer": 5, "softwar": 5, "engin": [5, 40, 70, 71, 73], "littl": [5, 15, 71, 73], "under": [5, 10, 14, 24], "15": [5, 70, 71], "profession": [5, 7], "experi": [5, 7, 15, 70, 71, 73], "wide": 5, "expertis": 5, "embed": [5, 7, 10, 12, 13, 14, 15, 18, 21, 22, 27, 33, 70, 71], "comput": [5, 10, 11, 14, 20, 21, 22, 23, 25, 26, 28, 29, 30, 32, 34, 36, 37, 39, 40, 69, 70, 71, 73], "coupl": 5, "gotten": [5, 14], "ml": [5, 70, 71, 73], "especi": [5, 14, 70, 71], "ai": [5, 15, 18, 38, 69, 71], "safeti": 5, "nine": 5, "march": 5, "chat": [5, 38, 71, 72], "bit": [5, 15, 70, 71], "he": [5, 71], "ask": 5, "might": [5, 10, 12, 13, 70], "interest": [5, 7, 12, 13, 14, 70, 71], "take": [5, 7, 10, 14, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 40, 69, 70, 71, 73], "role": 5, "maintain": 5, "basi": [5, 12, 14], "april": 5, "far": [5, 14, 69, 70, 71], "pretti": [5, 10, 14, 69, 70, 71], "mani": [5, 14, 24, 39, 40, 63, 70, 71], "kind": [5, 10, 70, 71], "address": 5, "everi": [5, 10, 12, 13, 14, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 40, 63, 69, 70, 71], "await": 5, "repli": 5, "total": [5, 70, 71], "around": [5, 6, 10, 12, 13, 14, 37, 40, 68, 70, 71], "30": [5, 70, 71, 72], "20": [5, 69, 70, 71, 72], "pr": 5, "were": [5, 10, 14, 15, 35, 69, 70, 71, 73], "limit": [5, 12, 13, 14, 70], "llama": [5, 7, 15, 38, 42, 72], "quantiz": [5, 15], "hookedsaetransform": 5, "brand": 5, "class": [5, 10, 11, 12, 13, 14, 15, 16, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 37, 38, 39, 63, 69], "splice": 5, "autoencod": 5, "two": [5, 6, 11, 12, 32, 36, 38, 40, 69, 70, 71], "goal": [5, 70, 71, 73], "posit": [5, 7, 10, 12, 13, 14, 15, 18, 21, 27, 30, 33, 35, 36, 37, 38, 40, 69, 70, 71], "while": [5, 12, 13, 14, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 70, 71], "remain": [5, 10, 14, 37, 71], "power": [5, 71], "who": 5, "push": 5, "second": [5, 14, 35, 70, 71], "base": [5, 7, 10, 11, 12, 13, 14, 15, 16, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 37, 38, 39, 63, 68, 69, 70, 71, 72], "easier": [5, 14, 35, 70, 71, 73], "futur": 5, "llm": 5, "continu": [5, 40, 70, 71], "acceler": [5, 70, 71], "feel": [5, 70, 71, 73], "massiv": [5, 70, 71], "amount": [5, 10, 15, 70], "momentum": [5, 63], "moment": [5, 18, 19, 24], "hope": [5, 71], "carri": 5, "over": [5, 10, 14, 40, 69, 70, 71], "background": [5, 70], "know": [5, 7, 10, 12, 13, 70, 71], "talk": [5, 71], "ensur": [5, 14, 70], "meet": 5, "person": [5, 70], "spoken": 5, "dozen": 5, "commun": 5, "happen": [5, 70, 71], "appoint": 5, "curiou": 5, "hear": 5, "anyon": [5, 35], "tool": [5, 7, 71, 73], "absolut": [5, 6, 12, 13, 14, 15, 18, 27, 36, 69, 70, 71], "beginn": 5, "complet": [5, 10, 18, 70, 71], "expert": [5, 15], "Not": [5, 18, 19, 24, 26], "idea": [5, 14, 18, 40, 70, 71, 73], "evolv": 5, "biggest": [5, 10], "previous": [5, 70], "offici": [5, 38, 71], "instanc": [5, 12, 13, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 36, 37, 70], "compat": [5, 14, 38], "through": [5, 14, 70, 71], "forward": [5, 10, 12, 13, 14, 15, 18, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 37, 39, 71], "start": [5, 10, 13, 14, 15, 18, 40, 69, 70, 71], "todai": [5, 71, 73], "assur": 5, "abl": [5, 70, 71], "upgrad": 5, "worri": 5, "There": [5, 6, 10, 12, 13, 36, 38, 69, 70, 71, 73], "right": [5, 12, 13, 14, 18, 19, 40, 69, 70, 71], "move_model": [5, 10], "activationcach": [5, 8, 9, 12, 13, 14, 36, 40, 69, 70, 71], "function": [5, 10, 12, 13, 14, 15, 18, 20, 21, 22, 23, 24, 25, 26, 28, 29, 32, 34, 36, 37, 38, 40, 63, 65, 66, 68, 69, 71, 73], "cache_al": [5, 37], "hook_point": [5, 8, 9, 14, 71], "keep": [5, 10, 14, 37, 70, 71, 73], "thing": [5, 11, 14, 15, 18, 40, 70, 71, 73], "simpl": [5, 70, 71], "howev": [5, 10, 14, 24, 35, 70, 71], "them": [5, 10, 14, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 40, 69, 70, 71], "adapt": [5, 71], "awai": [5, 14, 70], "along": [5, 14, 18, 69, 70, 71], "anyth": [5, 35, 70], "mark": [5, 70, 71], "whenev": [5, 37], "someth": [5, 14, 35, 70], "becom": [5, 70, 71], "promin": 5, "sort": [5, 71], "slip": 5, "scenario": 5, "situat": 5, "handl": [5, 14, 37], "persist": [5, 37], "allow": [5, 14, 36, 40, 69, 70, 71], "reli": 5, "interrupt": 5, "still": [5, 37, 70], "encourag": [5, 18, 69], "everyon": 5, "period": [5, 70, 71], "ey": 5, "don": [5, 10, 12, 13, 14, 15, 16, 35, 69, 70, 71, 73], "imagin": [5, 70], "often": [5, 10, 14, 15, 38, 69, 70, 71], "save": [5, 10, 14, 15, 36, 63, 69, 70, 71], "troubl": [5, 10], "move": [5, 10, 12, 13, 14, 40, 70, 71], "three": [5, 14, 40, 69, 70], "timefram": 5, "plan": 5, "state": [5, 10, 14, 31, 37, 70, 71, 73], "tracker": 5, "categor": 5, "easi": [5, 10, 14, 69, 70, 71, 73], "date": [5, 14], "below": [5, 12, 13, 14, 70], "draft": 5, "our": [5, 33, 70, 71], "priorit": 5, "feedback": [5, 69, 70, 71, 73], "surfac": 5, "other": [5, 10, 12, 14, 18, 19, 24, 36, 37, 38, 40, 70], "improv": [5, 69, 70, 71], "achiev": [5, 10, 71], "diagnos": 5, "variou": [5, 15, 37, 70, 71], "area": 5, "memori": [5, 10, 11, 12, 13, 14, 15, 70, 71], "leak": 5, "occur": [5, 70], "seem": [5, 14, 15, 35, 38, 70, 71], "refer": [5, 10, 14, 19, 37, 70, 71], "properli": [5, 35, 70], "thu": 5, "caus": 5, "garbag": 5, "collect": 5, "correctli": [5, 14], "identifi": [5, 7, 14, 40, 70, 71], "proper": 5, "overal": [5, 70], "deal": [5, 14, 18, 69, 70], "larger": [5, 10, 35, 70, 71], "task": [5, 7, 12, 14, 15, 35, 40, 63, 70], "explor": [5, 69, 71], "abil": [5, 10, 40, 70], "batch": [5, 10, 12, 13, 14, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 31, 32, 33, 34, 35, 37, 39, 40, 63, 66, 67, 69, 70, 71], "process": [5, 14, 15, 38, 69, 70, 71], "alreadi": [5, 10, 14, 69, 70, 71], "share": 5, "well": [5, 7, 14, 16, 37, 40, 69, 70, 71], "togeth": [5, 14, 69, 70, 71], "separ": [5, 12, 13, 14, 15, 37, 67, 69, 70, 71], "volunt": 5, "said": [5, 71], "submiss": 5, "discuss": [5, 70], "think": [5, 14, 35, 69, 70, 71], "few": [5, 10, 12, 13, 70, 71], "week": 5, "confus": [5, 70, 71], "among": 5, "calcul": [5, 10, 11, 12, 13, 14, 15, 18, 24, 36, 67, 69, 70, 71], "match": [5, 10, 14, 36, 66, 70], "huggingfac": [5, 12, 13, 14, 15, 35, 38, 66, 69, 71, 73], "solv": [5, 37, 70, 71, 73], "systemat": 5, "submit": 5, "show": [5, 7, 14, 16, 36, 70, 71, 73], "order": [5, 18, 40, 69, 70], "allevi": 5, "build": [5, 37, 71, 73], "spit": 5, "tabl": [5, 70, 71], "u": [5, 11, 14, 40, 69, 70, 71], "snapshot": 5, "store": [5, 10, 14, 15, 24, 37, 39, 40, 63, 70, 71], "repo": [5, 70], "regener": 5, "cumul": [5, 14, 69], "valu": [5, 7, 10, 11, 12, 13, 14, 15, 18, 24, 30, 31, 33, 36, 38, 39, 40, 69, 70, 71, 73], "creat": [5, 7, 10, 14, 18, 37, 70, 71], "robust": [5, 10, 70], "big": [5, 36, 38, 69, 70, 71], "famili": [5, 15, 71], "hard": [5, 14, 70, 71], "even": [5, 7, 11, 14, 15, 18, 35, 38, 70, 71, 73], "smallest": [5, 71], "thought": [5, 10, 70, 71], "thrown": 5, "topic": [5, 7], "best": [5, 7, 14, 70, 71], "guess": [5, 70], "reason": [5, 18, 19, 24, 70, 71], "untrain": 5, "eg": [5, 10, 14, 35, 40, 69, 70, 71], "randomli": [5, 14, 15, 71], "weight": [5, 7, 12, 13, 14, 15, 18, 24, 26, 63, 66, 69, 70, 71, 73], "verifi": [5, 7, 70, 71], "load": [5, 10, 12, 13, 14, 15, 35, 38, 69, 70, 73], "result": [5, 10, 12, 13, 14, 15, 21, 30, 33, 36, 38, 40, 69, 70, 71, 73], "accur": 5, "sens": [5, 11, 37, 70, 71], "consist": [5, 14, 70, 71], "sampl": [5, 14, 35, 69], "size": [5, 10, 14, 15, 18, 35, 63, 69, 70, 71], "against": [5, 70], "bite": 5, "success": [5, 70], "turn": [5, 10, 14, 69, 70, 71], "effici": [5, 11, 18, 69, 71], "proof": [5, 70], "put": [5, 70, 71], "strong": 5, "opinion": 5, "most": [5, 10, 14, 37, 69, 70, 71, 73], "roundtabl": 5, "wrapper": [5, 10, 12, 13, 14, 40, 68, 71], "plugin": 5, "addit": [5, 7, 12, 13, 14, 70], "outsid": 5, "publish": 5, "themselv": [5, 14], "final": [5, 6, 10, 12, 13, 14, 15, 18, 69, 70, 71], "overhaul": 5, "composit": [5, 14, 70, 71], "util": [5, 8, 9, 10, 11, 14, 36, 37, 38, 63, 70, 71], "isol": [5, 70], "rapidli": 5, "itself": [5, 36, 69, 70], "none": [5, 10, 12, 13, 14, 15, 16, 18, 19, 20, 21, 24, 25, 27, 28, 30, 31, 33, 35, 36, 37, 38, 40, 63, 68, 69, 70, 71], "pain": [5, 71], "rel": [5, 15, 30, 70], "grow": 5, "exponenti": 5, "whole": [5, 70, 71], "explod": 5, "section": [5, 70, 71], "relev": [5, 14, 15, 18, 40, 69, 70, 71], "skip": [5, 14, 70, 71], "setup": [5, 14, 37, 40], "act": [5, 37, 40, 69, 70, 71], "vast": 5, "due": [5, 6, 14, 71], "potenti": 5, "mismatch": [5, 36], "between": [5, 10, 14, 15, 18, 36, 40, 69, 70, 71, 73], "meant": 5, "repres": [5, 11, 12, 15, 32, 36, 40, 69, 70, 71], "updat": [5, 7, 14, 39, 40, 68, 70, 71], "readi": 5, "sent": 5, "justifi": 5, "bug": [5, 7, 10, 15, 69, 71], "fix": [5, 37, 70, 71], "exist": [5, 36, 70, 71], "split": [5, 14, 18, 38, 69, 70, 71], "group": [5, 15, 18, 24], "call": [5, 10, 12, 13, 14, 15, 20, 21, 22, 23, 24, 25, 26, 28, 29, 32, 34, 37, 38, 69, 70, 71], "again": [5, 70], "piec": [5, 70], "tradition": 5, "everyth": [5, 37, 40, 69, 71], "That": [5, 70], "mock": 5, "spi": 5, "control": [5, 15, 40, 70, 71], "input": [5, 10, 12, 13, 14, 15, 24, 27, 32, 37, 38, 39, 40, 66, 67, 69, 70, 71], "side": [5, 11, 14], "effect": [5, 10, 14, 15, 40, 70, 71], "certain": [5, 15, 40], "logic": 5, "entir": [5, 10, 14, 40, 70], "rule": 5, "incredibli": [5, 71], "cannot": [5, 14, 69, 71], "origin": [5, 15, 16, 18, 70, 71], "pass": [5, 10, 14, 15, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 32, 33, 34, 36, 37, 38, 39, 69, 71], "bunch": [5, 10, 12, 13, 14, 37, 70, 71], "Being": [5, 70], "live": [5, 12, 13, 14, 70, 71], "ci": 5, "report": 5, "great": [5, 7, 71, 73], "get": [5, 7, 10, 12, 13, 14, 15, 16, 18, 35, 36, 37, 40, 68, 70, 71, 73], "review": 5, "meaning": [5, 15, 40], "than": [5, 6, 10, 14, 15, 18, 20, 35, 36, 37, 40, 69, 70, 71], "ever": [5, 69], "substanti": 5, "let": [5, 15, 69, 70, 71, 73], "shift": [5, 40, 71], "favor": 5, "individu": [5, 7, 10, 14, 18, 70], "3000": 5, "18": [5, 69, 70, 71, 72], "distinct": 5, "either": [5, 14, 36, 37, 38, 40, 70, 71], "interdepend": 5, "least": [5, 40, 71], "depend": [5, 10, 14, 71], "its": [5, 10, 14, 18, 37, 70, 71, 73], "own": [5, 7, 70, 71], "realli": [5, 35, 36, 69, 70, 71], "anywher": [5, 40], "els": [5, 14, 15, 18, 38, 69, 70, 71], "mlp": [5, 10, 12, 13, 14, 15, 20, 33, 40, 69, 70, 71], "exactli": [5, 14, 38, 66, 70, 71], "thank": [5, 71], "excit": 5, "standpoint": 5, "perspect": [5, 71], "comparison": [5, 36], "worth": [5, 7, 10, 70, 71], "enabl": [5, 10, 14, 70, 71, 73], "huge": 5, "impact": [5, 10], "bring": 5, "realiti": [5, 70], "semver": 5, "older": [5, 71], "log": [5, 14, 63, 69, 70, 71], "data": [5, 6, 7, 14, 35, 69, 70, 71], "expos": [5, 14, 73], "properti": [5, 11, 12, 13, 14, 18, 24, 69, 70, 71], "minor": [5, 70], "bump": [5, 71], "whatsoev": 5, "With": [5, 18, 71], "fact": [5, 14, 70, 71, 73], "discov": 5, "earlier": [5, 70, 71], "extent": 5, "probabl": [5, 7, 14, 35, 36, 40, 69, 70, 71], "regardless": [5, 69], "stand": 5, "reliabl": 5, "17": [5, 70, 71], "possibli": 5, "easiest": [5, 70], "fresh": 5, "consum": [5, 10, 11], "top": [6, 14, 69, 70, 71], "k": [6, 10, 11, 12, 13, 14, 16, 18, 24, 40, 69, 70, 71], "gate": [6, 14], "hidden": [6, 15, 31, 71], "amplifi": 6, "greatli": [6, 71, 73], "select": [6, 10, 69, 70, 71], "lead": [6, 7, 12, 13, 37, 69, 71], "higher": [6, 14, 70], "normal": [6, 10, 14, 15, 26, 69, 70, 71, 73], "varianc": [6, 70], "test": [6, 7, 14, 35, 36, 69, 70, 71], "half": [6, 11, 12, 13, 14, 18, 35, 71], "precis": [6, 36, 40, 70, 71], "deviat": [6, 15, 71], "compar": [6, 35, 71, 73], "2e": 6, "mitig": 6, "disabl": [6, 14, 36, 37, 70], "preprocess": [6, 12, 13, 70], "option": [6, 10, 12, 13, 14, 15, 16, 18, 19, 20, 21, 24, 25, 27, 28, 30, 31, 33, 35, 36, 37, 38, 39, 40, 63, 68, 69], "from_pretrained_no_process": [6, 14], "increas": [6, 40, 70, 71], "colab": [7, 70, 71, 73], "blob": [7, 66], "ipynb": 7, "causal": [7, 12, 15, 40, 70, 71], "intervent": [7, 40, 70, 71], "matter": [7, 14, 40, 70, 71], "produc": [7, 14, 40, 70], "incomplet": 7, "gradient": [7, 10, 37, 63, 71], "approxim": [7, 70, 71], "bad": [7, 14], "residu": [7, 10, 12, 13, 14, 15, 18, 31, 33, 34, 40, 71], "stream": [7, 10, 12, 13, 14, 15, 31, 33, 40, 69, 71], "after": [7, 10, 14, 15, 20, 26, 37, 63, 70, 71, 73], "demonstr": [7, 16, 70, 71], "focus": [7, 70, 71], "less": [7, 14, 18, 70], "rigor": [7, 70, 71], "grasp": 7, "steal": 7, "liber": [7, 37], "phenomenon": 7, "memoris": 7, "minimis": 7, "loss": [7, 10, 14, 35, 37, 40, 63, 69, 70, 71], "longer": 7, "generalis": [7, 70, 71], "sharp": [7, 71], "decreas": [7, 18, 69, 70], "modular": [7, 69], "grok": 7, "light": 7, "explan": [7, 40, 70], "ll": [7, 14, 36, 70, 71], "pair": [7, 11, 14, 18, 36, 69, 70, 71], "seri": [7, 10, 71], "detector": [7, 36], "detect": [7, 36, 70, 71], "sever": [7, 10, 14, 69, 70, 71], "custom": [7, 14, 15, 21, 35, 37, 69, 70, 71], "algorithm": [7, 11, 15, 71, 73], "interact": [7, 65, 70, 71], "neuroscop": [7, 71], "hacki": [7, 69], "web": [7, 71], "visualis": [7, 70], "front": 7, "visual": [7, 12, 13, 71], "dynam": [7, 15, 71], "convert": [7, 12, 13, 14, 38, 69, 70, 71], "meta": [7, 14, 38, 69, 70, 71, 72], "until": [7, 10, 14, 37, 70, 71], "multi": [7, 10, 69, 71], "gpu": [7, 10, 11, 12, 13, 14, 70, 71], "access": [7, 10, 15, 37, 69, 70], "No": [7, 71], "previou": [7, 10, 14, 31, 33, 36, 70, 71], "port": 7, "excel": [7, 10, 70, 71, 73], "sequenc": [7, 12, 13, 14, 15, 18, 32, 35, 36, 37, 40, 69, 70, 71], "investig": [7, 10, 14, 36, 70, 71], "svd": [7, 11, 14, 16, 71], "conjectur": 7, "post": [7, 10, 15, 16, 70, 71], "singular": [7, 11, 14, 16, 71], "decomposit": [7, 10, 11, 14, 70, 71], "matric": [7, 11, 12, 13, 14, 16, 18, 19, 24, 66, 70, 71], "surprisingli": 7, "reproduc": [7, 15, 36], "further": [7, 10, 14, 69, 70, 71], "tracr": 7, "cool": 7, "deepmind": 7, "compil": 7, "program": [7, 71, 73], "rasp": 7, "jax": 7, "form": [7, 10, 11, 14, 40, 70, 71], "pytorch": [7, 14, 15, 35, 37, 71], "brows": 8, "first": [8, 10, 14, 15, 35, 38, 40, 69, 70, 71], "submodul": 8, "factoredmatrix": [8, 9, 12, 13, 18, 69, 71], "hookedencod": [8, 9, 68], "hookedencoderdecod": [8, 9, 68], "hookedtransformerconfig": [8, 9, 14, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 38, 39, 44, 45, 51, 52, 58, 59, 60, 61, 62, 68], "svdinterpret": [8, 9], "eval": [8, 9, 71], "head_detector": [8, 9], "past_key_value_cach": [8, 9], "subpackag": 8, "pretrain": [8, 9, 10, 12, 13, 14, 15, 35, 38, 69, 70, 71], "core": [10, 14, 70, 71, 73], "varieti": [10, 71], "helper": [10, 14, 18, 35, 37, 40, 69, 71], "skim": 10, "method": [10, 12, 13, 14, 15, 37, 38, 39, 69, 70, 71], "back": [10, 15, 18, 19, 24, 71], "cache_dict": 10, "dict": [10, 12, 13, 14, 15, 18, 19, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 32, 33, 34, 35, 36, 37, 38, 69], "str": [10, 12, 13, 14, 15, 18, 19, 24, 30, 35, 36, 37, 38, 39, 40, 63, 68, 69, 70, 71], "tensor": [10, 11, 12, 13, 14, 16, 18, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 36, 37, 39, 40, 66, 67, 69, 70, 71], "has_batch_dim": 10, "bool": [10, 12, 13, 14, 15, 30, 31, 35, 36, 37, 38, 39, 40, 63, 69], "run_with_cach": [10, 12, 13, 14, 37, 70, 71], "particular": [10, 16, 36, 70, 71], "behaviour": [10, 14, 70, 71], "modal": 10, "step": [10, 14, 15, 38, 63, 69, 70, 71], "respons": [10, 15, 70], "prompt": [10, 14, 18, 35, 40, 69, 70, 71], "chicken": 10, "road": [10, 69], "specif": [10, 12, 13, 14, 18, 36, 40, 70, 71], "sublay": 10, "commonli": 10, "fall": 10, "categori": [10, 70], "dla": 10, "_logit": 10, "residual_stream": 10, "label": [10, 14, 15, 18, 19, 24, 38, 70, 71], "decompose_resid": [10, 70], "return_label": [10, 70], "0": [10, 12, 13, 14, 15, 16, 18, 27, 32, 36, 38, 63, 69, 70, 71, 72], "emb": [10, 17, 21, 27, 32, 69, 71], "pos_emb": [10, 15, 17, 71], "0_attn_out": 10, "proceed": 10, "space": [10, 12, 14, 69, 70, 71], "logit_attr": 10, "shape": [10, 12, 14, 18, 19, 24, 31, 32, 33, 40, 69, 70, 71], "torch": [10, 12, 13, 14, 15, 18, 24, 27, 31, 33, 37, 38, 40, 66, 68, 69, 70, 71], "10": [10, 14, 16, 69, 70, 71, 72], "7": [10, 35, 70, 71, 72], "most_important_component_idx": 10, "argmax": [10, 70], "3_attn_out": 10, "dig": [10, 70, 71, 73], "granular": 10, "get_full_resid_decomposit": 10, "stack": [10, 12, 13, 14, 40, 69, 70, 71], "equal": [10, 15], "struggl": 10, "construct": [10, 12, 13], "joke": 10, "trivial": 10, "accumulated_resid": [10, 70], "footgun": [10, 37], "sourc": [10, 14, 15, 18, 35, 40, 73], "track": [10, 70], "index": [10, 12, 13, 14, 15, 16, 18, 19, 24, 38, 40, 68, 69, 70, 71], "dimens": [10, 14, 15, 18, 24, 25, 28, 37, 40, 66, 67, 69, 70, 71], "vector": [10, 11, 14, 16, 18, 40, 70, 71], "q": [10, 12, 13, 14, 18, 24, 40], "z": [10, 14, 24, 36, 40, 70, 71], "po": [10, 12, 13, 14, 18, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 40, 67, 69, 70, 71], "head_index": [10, 14, 16, 18, 19, 24, 25, 26, 30, 31, 40, 67, 70, 71], "d_head": [10, 12, 13, 14, 15, 18, 19, 24, 38, 39, 67, 69, 70, 71, 72], "pattern": [10, 12, 13, 14, 18, 19, 24, 36, 40, 71], "softmax": [10, 14, 15, 18, 19, 24, 26, 69, 71], "attn_scor": [10, 15, 18, 19], "pre": [10, 14, 15, 18, 19, 24, 26, 29, 36, 69], "query_po": [10, 18, 19, 24, 70], "key_po": [10, 18, 19, 24, 70], "d_model": [10, 12, 13, 14, 15, 18, 19, 20, 22, 23, 24, 25, 26, 27, 28, 31, 33, 34, 38, 67, 69, 70, 71, 72], "mid": [10, 70], "solu_ln": [10, 15], "layernorm": [10, 12, 13, 14, 15, 20, 25, 26, 28, 69, 70], "d_mlp": [10, 12, 13, 14, 15, 26, 38, 69, 71, 72], "resid_pr": [10, 15, 20, 31, 33, 40, 70, 71], "resid_mid": [10, 40], "resid_post": [10, 15, 70], "attn_out": [10, 14, 15, 40, 70], "mlp_out": [10, 14, 15, 40, 70], "ln": [10, 14, 15, 70, 71], "lnpre": [10, 15], "scale": [10, 14, 15, 18, 19, 24, 69, 70, 71], "sometim": [10, 35, 70], "miss": [10, 70], "becaus": [10, 11, 12, 13, 14, 15, 18, 35, 69, 70, 71, 73], "appli": [10, 14, 15, 18, 20, 33, 37, 40, 69, 70, 71], "remove_batch_dim": [10, 37, 69, 71], "batch_siz": [10, 12, 32, 35, 37, 39, 63, 70, 71], "annot": [10, 71], "layers_cov": 10, "queri": [10, 12, 13, 14, 15, 18, 19, 24, 30, 36, 40, 71], "batch_and_pos_dim": 10, "ve": [10, 14, 18, 35, 37, 70, 73], "slice": [10, 37, 69, 70], "dictionari": [10, 12, 13, 14, 15, 36, 37, 38, 69, 71], "whether": [10, 12, 14, 15, 18, 32, 35, 37, 38, 40, 63, 69, 70, 71], "int": [10, 11, 12, 13, 14, 15, 16, 18, 19, 21, 23, 24, 25, 27, 28, 30, 31, 32, 33, 35, 36, 37, 38, 39, 40, 63, 68, 69, 70, 71], "incl_mid": [10, 70], "fals": [10, 12, 13, 14, 15, 30, 35, 36, 37, 38, 39, 40, 63, 69, 70, 71], "apply_ln": [10, 70], "pos_slic": [10, 37, 70], "union": [10, 12, 13, 14, 15, 16, 18, 19, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 32, 33, 34, 36, 37, 38, 39, 40, 68, 69, 70], "tupl": [10, 11, 12, 13, 14, 18, 24, 36, 37, 40, 69, 71], "ndarrai": [10, 14, 37, 69], "mlp_input": [10, 14], "float": [10, 11, 12, 13, 14, 15, 18, 20, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 33, 34, 36, 38, 39, 40, 63, 66, 67, 69, 70, 71], "accumul": [10, 14, 70], "sub": [10, 71], "www": 10, "lesswrong": 10, "ackrb8wdpdan6v6ru": 10, "believ": [10, 35, 70], "vocabulari": [10, 15, 70, 71], "rememb": 10, "norm": [10, 11, 14, 15, 21, 25, 26, 28, 29, 38, 63, 70, 71], "decod": [10, 13, 15, 31], "therefor": [10, 14, 69], "multipli": [10, 14, 18, 19, 36, 66, 70, 71], "unembed": [10, 12, 13, 14, 70, 71], "matrix": [10, 11, 12, 13, 14, 15, 16, 18, 24, 36, 69, 70], "w_u": [10, 12, 13, 14, 70, 71], "broken": [10, 37, 69, 70, 71], "down": [10, 14, 18, 19, 24, 70, 71], "einop": [10, 70, 71], "einsum": [10, 70, 71], "panda": [10, 40], "pd": [10, 40], "devic": [10, 12, 13, 14, 15, 16, 18, 30, 35, 37, 38, 39, 63, 64, 70, 71], "answer_token": [10, 70], "to_single_token": [10, 14, 70, 71], "2975": 10, "accum_resid": 10, "last_token_accum": 10, "9": [10, 69, 70, 71, 72], "64": [10, 38, 69, 71, 72], "50257": [10, 38, 71, 72], "layers_unembed": 10, "d_vocab": [10, 12, 13, 14, 15, 16, 38, 40, 69, 71, 72], "rank": [10, 11, 14, 18, 69, 70, 71], "correct": [10, 40, 68, 69, 70, 71], "sorted_indic": 10, "argsort": 10, "dim": [10, 14, 69, 70, 71], "descend": [10, 71], "rank_answ": 10, "nonzero": 10, "as_tupl": 10, "0_pre": 10, "4442": 10, "1_pre": [10, 70], "382": 10, "2_pre": 10, "982": 10, "3_pre": 10, "1160": 10, "4_pre": 10, "408": 10, "5_pre": 10, "145": 10, "6_pre": 10, "78": 10, "7_pre": 10, "387": 10, "final_post": 10, "6": [10, 14, 38, 69, 70, 71, 72], "dtype": [10, 12, 13, 14, 15, 18, 38, 68, 70, 71], "int64": [10, 69], "exclud": [10, 36], "n_layer": [10, 12, 13, 14, 15, 38, 40, 68, 70, 71, 72], "immedi": [10, 19, 69, 70, 71], "indic": [10, 12, 13, 32, 37, 40, 69, 70, 71], "taken": [10, 14, 71], "l": [10, 12, 13, 14, 70, 71], "noth": [10, 12, 13, 14, 37, 69, 70, 71], "essenti": [10, 14, 70, 71, 73], "rather": [10, 14, 15, 20, 40, 69, 70, 71], "graph": [10, 70, 71], "apply_ln_to_stack": [10, 14, 69, 70], "residual_stack": [10, 70], "num_compon": 10, "batch_slic": 10, "batch_and_pos_dims_out": 10, "treat": [10, 14, 15, 70, 71], "factor": [10, 11, 14, 15, 70], "simul": [10, 14, 70, 71], "global": [10, 18, 19, 24, 30, 37, 69, 70, 71], "element": [10, 14, 18, 36, 40, 69, 71], "rmsnorm": [10, 15, 28], "unchang": [10, 12, 13, 14, 69, 70, 71], "whose": [10, 12, 13, 14, 35, 69, 70], "trail": [10, 11, 69], "assum": [10, 12, 13, 14, 15, 25, 28, 32, 37, 40, 63, 69, 70], "hook_scal": [10, 69, 70, 71], "unemb": [10, 14, 15, 17, 70, 71], "map": [10, 12, 13, 14, 15, 18, 36, 37, 70, 71], "ie": [10, 12, 13, 14, 15, 18, 38, 40, 69, 70, 71], "ln2": [10, 33, 69, 71], "ln1": [10, 15, 33, 69, 71], "ln_final": [10, 14, 70, 71], "apply_slice_to_batch_dim": 10, "compute_head_result": 10, "sum": [10, 11, 14, 15, 21, 36, 69, 70, 71], "plu": 10, "b_o": [10, 12, 13, 14, 71], "intend": [10, 15, 69], "use_attn_result": [10, 14, 15], "forget": 10, "liter": [10, 12, 13, 14, 16, 36, 37, 40], "incl_emb": 10, "decompos": 10, "incl": 10, "expand_neuron": 10, "bias": [10, 12, 13, 14, 15, 63, 70], "expand": [10, 14, 24], "get_neuron_result": 10, "neuron_slic": 10, "num_neuron": 10, "subset": [10, 15, 35, 70, 71], "specifi": [10, 12, 13, 14, 15, 24, 35, 36, 37, 68, 69, 71], "expens": [10, 11], "cheap": 10, "hook_emb": [10, 69, 71], "hook_pos_emb": [10, 71], "block": [10, 14, 15, 18, 19, 20, 24, 31, 33, 40, 69, 70, 71], "hook_resid_pr": [10, 71], "incorrect_token": [10, 70], "typic": [10, 12, 14, 32, 36, 70, 71], "revers": [10, 11, 40, 69, 70, 71, 73], "dot": [10, 14, 18, 19, 69], "product": [10, 11, 12, 13, 18, 19, 71], "incorrect": [10, 14, 40, 70, 71], "arxiv": [10, 14, 15, 18, 24, 32, 35], "org": [10, 14, 15, 18, 24, 32, 35, 71, 73], "ab": [10, 11, 14, 24, 36, 70, 71], "2211": [10, 35], "00593": [10, 35], "john": [10, 70, 71], "mari": [10, 70, 71], "went": [10, 70, 71], "shop": [10, 70, 71], "gave": [10, 35, 70, 71], "bag": [10, 70], "choos": [10, 70, 71], "final_ln": 10, "residual_stack_item": 10, "dure": [10, 24, 37, 39, 71, 73], "stack_activ": 10, "activation_nam": [10, 40, 70], "sublayer_typ": 10, "flexibl": 10, "given": [10, 11, 12, 13, 14, 16, 36, 37, 38, 40, 68, 69, 70, 71], "get_act_nam": [10, 69, 70, 71], "infer": [10, 14, 26, 40, 70, 71], "incl_remaind": 10, "stack_head_result": [10, 70], "axi": [10, 18, 40, 69, 70, 71], "n_head": [10, 12, 13, 14, 15, 18, 24, 38, 39, 40, 69, 70, 71, 72], "notat": [10, 70], "l0h0": 10, "stack_neuron_result": 10, "l0n0": 10, "super": [10, 14, 69, 71], "short": [10, 69, 70, 71, 73], "mostli": [10, 70, 71], "finish": [10, 14, 69, 70, 71], "oper": [10, 70, 71], "slower": 10, "unless": [10, 14, 15, 35, 71], "deprec": 10, "toggle_autodiff": 10, "toggl": [10, 14], "autodiff": [10, 71], "set_grad_en": [10, 70, 71], "danger": 10, "off": [10, 14, 35, 69, 70, 71], "realis": [10, 30, 70], "downstream": 10, "delet": [10, 69, 70], "stick": [10, 70], "mess": [10, 14, 69, 71], "inference_mod": 10, "decor": 10, "similar": [10, 12, 13, 14, 20, 24, 36, 70, 71], "requires_grad": 10, "eigenvalu": 11, "ldim": [11, 71], "mdim": [11, 71], "rdim": [11, 71], "leading_dim": [11, 69], "ba": 11, "vh": [11, 14], "collapse_l": 11, "collaps": [11, 70, 71], "left": [11, 14, 18, 69, 70, 71, 73], "orthogon": [11, 14], "self": [11, 12, 13, 14, 18, 69, 71], "collapse_r": 11, "analog": [11, 70, 71], "apart": [11, 69, 70, 71], "zero": [11, 14, 18, 36, 69, 70, 71], "bav": 11, "kv": 11, "abav": 11, "kav": 11, "av": 11, "eigenvector": [11, 71], "get_corn": [11, 69, 70], "make_even": 11, "sqrt": [11, 14, 15, 69], "diag": 11, "equival": [11, 14, 18, 70, 71], "factoris": [11, 14, 18, 71], "row": [11, 14, 40], "col": 11, "ndim": 11, "frobeniu": [11, 71], "squar": [11, 28, 29, 69, 71], "m": [11, 18, 66, 69, 70, 71], "st": 11, "transpos": [11, 69], "obviou": [11, 14, 70], "unsqueez": [11, 69], "hook": [12, 13, 14, 15, 19, 20, 21, 22, 23, 25, 26, 27, 28, 29, 32, 33, 34, 37, 70], "encod": [12, 13, 18, 31, 69, 71], "contain": [12, 13, 14, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 32, 33, 34, 35, 36, 37, 38, 39, 69, 70, 71], "bert": [12, 20, 21, 22, 32, 38, 42, 71, 72], "move_to_devic": [12, 13, 14], "kwarg": [12, 13, 14, 38, 69, 70, 71], "hookedrootmodul": [12, 13, 14, 37, 71], "hookpoint": [12, 13, 14, 37, 70, 71], "inherit": [12, 13, 71], "mvp": 12, "mask": [12, 13, 14, 18, 22, 27, 31, 33, 69, 70], "mlm": [12, 22], "next": [12, 14, 69, 70, 71], "sentenc": [12, 13, 14, 22, 32, 36, 70, 71], "nsp": 12, "dropout": [12, 13], "inconsist": [12, 13, 16], "fine": [12, 13, 71], "fold": [12, 13, 14, 18, 26, 38, 70], "ov": [12, 13, 14, 16, 18, 70, 71], "o": [12, 13, 66, 71], "qk": [12, 13, 14, 18, 70], "w_e": [12, 13, 14, 71], "conveni": [12, 13, 14, 15, 37, 69, 71], "w_e_po": [12, 14], "n_ctx": [12, 14, 15, 18, 38, 71, 72], "concaten": [12, 14, 69, 70, 71], "w_po": [12, 13, 14, 71], "overcomplet": [12, 14], "w_k": [12, 13, 14, 15, 18, 24, 71], "w_o": [12, 13, 14, 18, 19, 70, 71], "w_q": [12, 13, 14, 18, 24, 71], "w_v": [12, 13, 14, 18, 24, 71], "w_in": [12, 13, 14, 16, 71], "w_out": [12, 13, 14, 16, 71], "all_head_label": [12, 13, 14], "format": [12, 13, 14, 37, 70, 71], "h": [12, 13, 14, 70, 71], "b_k": [12, 13, 14, 18, 24, 71], "b_q": [12, 13, 14, 71], "b_u": [12, 13, 14, 70, 71], "bia": [12, 13, 14, 15, 18, 28, 29, 30, 66, 70, 71], "b_v": [12, 13, 14, 18, 24, 71], "b_in": [12, 13, 14, 71], "b_out": [12, 13, 14, 71], "buffer": [12, 13, 14], "modifi": [12, 13, 14], "cuda": [12, 13, 14, 15, 35, 38], "associ": [12, 13, 14, 37], "optim": [12, 13, 14, 63, 70], "return_typ": [12, 13, 14, 37, 70, 71], "token_type_id": [12, 21, 32], "one_zero_attention_mask": [12, 13], "binari": [12, 13, 32], "id": [12, 14, 15, 32], "belong": [12, 32], "cl": [12, 32, 71], "sep": [12, 32], "sequence_length": [12, 32, 36, 69], "attend": [12, 13, 15, 18, 19, 24, 69, 70, 71], "ignor": [12, 13, 14, 15, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 69, 71], "primarili": [12, 13], "pad": [12, 13, 14, 18, 27, 31, 33, 69, 70, 71], "shorter": [12, 13, 14, 71], "classmethod": [12, 13, 14, 15, 39, 69], "model_nam": [12, 13, 14, 15, 38, 71], "checkpoint_index": [12, 13, 14, 15, 38, 71], "checkpoint_valu": [12, 13, 14, 15, 38, 71], "hf_model": [12, 13, 14], "float32": [12, 13, 14, 15, 18, 38, 70], "from_pretrained_kwarg": [12, 13, 14], "bertformaskedlm": [12, 13], "unlik": [12, 13, 14, 40, 71], "mp": [12, 13, 14], "model_arg": [12, 13, 14, 37], "return_cache_object": [12, 13, 14], "otherwis": [12, 13, 14, 35, 36, 69], "device_or_dtyp": [12, 13, 14, 68], "print_detail": [12, 13, 14, 68, 69], "cast": [12, 13, 14], "non_block": [12, 13, 14], "memory_format": [12, 13, 14], "channels_last": [12, 13, 14], "Its": [12, 13, 14], "complex": [12, 13, 14, 15, 70, 71], "integr": [12, 13, 14, 38], "tri": [12, 13, 14, 70, 71, 73], "asynchron": [12, 13, 14], "respect": [12, 13, 14, 37, 69, 71], "host": [12, 13, 14, 38], "pin": [12, 13, 14], "desir": [12, 13, 14], "4d": [12, 13, 14], "keyword": [12, 13, 14, 37, 71], "argument": [12, 13, 14, 15, 37, 38, 69, 71], "xdoctest": [12, 13, 14], "ignore_w": [12, 13, 14], "non": [12, 13, 14, 15, 18, 35, 69, 70, 71], "determinist": [12, 13, 14, 69, 70], "nn": [12, 13, 14, 37, 71], "1913": [12, 13, 14], "3420": [12, 13, 14], "5113": [12, 13, 14], "2325": [12, 13, 14], "doubl": [12, 13, 14], "in_featur": [12, 13, 14], "out_featur": [12, 13, 14], "float64": [12, 13, 14], "env": [12, 13, 14], "torch_doctest_cuda1": [12, 13, 14], "gpu1": [12, 13, 14], "1914": [12, 13, 14], "5112": [12, 13, 14], "2324": [12, 13, 14], "float16": [12, 13, 14], "cdoubl": [12, 13, 14], "3741": [12, 13, 14], "j": [12, 13, 14, 15, 18, 38, 70, 71, 72], "2382": [12, 13, 14], "5593": [12, 13, 14], "4443": [12, 13, 14], "complex128": [12, 13, 14], "6122": [12, 13, 14], "1150": [12, 13, 14], "encoderdecod": [13, 15], "t5": [13, 15, 30, 31, 38, 42, 72], "decoder_input": 13, "decoder_po": 13, "usual": [13, 31], "fairli": [14, 70, 71], "extract": [14, 71], "harder": [14, 40, 70], "aim": [14, 70, 73], "simplifi": [14, 70, 71], "attach": [14, 71], "within": [14, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 36, 37, 40, 69, 70, 71], "inspect": [14, 70], "alter": 14, "facilit": 14, "deeper": 14, "pretrainedtokenizerbas": 14, "default_padding_sid": 14, "50": [14, 63, 71], "initialis": [14, 15], "although": [14, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37], "instanti": [14, 15, 71], "__init__": [14, 18, 19, 24, 25, 26, 28, 29, 37, 69, 71], "test_prompt": [14, 69, 70, 71], "w_gate": 14, "tokenizer_nam": [14, 15], "explicitli": [14, 15, 18, 35, 38, 40, 71], "n_devic": [14, 15, 38, 68], "greater": [14, 36], "multipl": [14, 36, 68, 69, 71], "accumulated_bia": 14, "include_mlp_bias": 14, "all_composition_scor": [14, 69], "score": [14, 15, 18, 19, 24, 36, 40, 70], "l1": 14, "h1": 14, "l2": 14, "h2": [14, 70], "upper": [14, 18], "triangular": [14, 36, 69, 71], "third": [14, 71], "pub": [14, 69], "2021": 14, "framework": [14, 18, 70, 71], "html": [14, 69, 70], "20abov": 14, "20diagram": 14, "20show": 14, "20q": 14, "2d": [14, 66, 69], "2c": 14, "20k": [14, 69], "20and": 14, "20v": 14, "2dcomposit": 14, "metric": [14, 36, 40, 70, 71], "center_unemb": [14, 70], "state_dict": 14, "center": [14, 15, 26, 28, 29, 70, 71], "subtract": [14, 36, 70], "translat": [14, 70, 71], "invari": 14, "prob": [14, 69, 70, 71], "slightli": [14, 69, 70], "misl": 14, "center_writing_weight": [14, 70, 71], "fold_layer_norm": [14, 38], "check_hooks_to_add": [14, 37], "hook_point_nam": [14, 37], "dir": [14, 37], "fwd": [14, 37], "is_perman": [14, 37], "prepend": [14, 15, 35, 37, 38, 69, 71], "overrid": [14, 15, 37, 38, 69], "fold_bias": 14, "center_weight": 14, "rm": [14, 15, 28, 29], "neighbour": 14, "further_com": [14, 15], "md": [14, 15], "fold_value_bias": 14, "alwai": [14, 15, 40, 70, 71], "constant": [14, 15, 18, 70, 71], "doesn": [14, 35, 69, 70, 71], "formal": 14, "b_o_new": 14, "b_o_origin": 14, "sum_head": 14, "b_v_head": 14, "w_o_head": 14, "loss_per_token": 14, "prepend_bo": [14, 15, 35, 38, 69, 70], "use_default_valu": 14, "padding_sid": [14, 69, 70], "start_at_lay": 14, "shortformer_pos_emb": [14, 18, 33], "attention_mask": [14, 18, 27, 31, 33, 39, 69], "stop_at_lay": 14, "past_kv_cach": [14, 27], "hookedtransformerkeyvaluecach": [14, 31, 33, 39], "flag": [14, 15, 35, 37, 40, 69, 70, 71], "entropi": [14, 69, 70, 71], "per": [14, 40, 69, 70, 71], "averag": [14, 35, 70, 71], "scalar": [14, 18, 37, 71], "default_prepend_bo": [14, 15, 35, 38, 69, 71], "bo": [14, 15, 36, 38, 69, 70, 71], "impli": 14, "usag": [14, 70], "accordingli": [14, 15, 18, 38, 70, 71], "lose": [14, 15, 38], "empir": [14, 15, 38, 40, 71], "inclus": 14, "neg": [14, 69, 70, 71], "shortform": [14, 15, 18, 33, 38], "positional_embedding_typ": [14, 15, 18], "stop": 14, "exclus": [14, 69], "etc": [14, 15, 40, 70, 71, 73], "frozen": [14, 39], "pai": [14, 18, 70], "okai": 14, "twice": [14, 35, 70, 71], "accident": [14, 37], "fold_ln": [14, 38, 70, 71], "refactor_factored_attn_matric": [14, 70], "automodelforcausallm": 14, "first_n_lay": [14, 38], "autoregress": [14, 63], "neo": [14, 18, 19, 24, 38, 42, 71, 72], "gptj": [14, 38, 42], "opt": [14, 38, 42, 71, 72], "solu": [14, 15, 38, 69, 71, 72], "checkpoint": [14, 15, 38, 63], "neelnanda": [14, 38], "stanford": [14, 15, 18, 19, 24, 38, 71, 72], "crfm": [14, 38, 71], "load_and_process_state_dict": 14, "alia": [14, 37, 38, 69, 71], "subsequ": [14, 38, 70, 71], "regular": [14, 18, 24], "batchnorm": [14, 70, 71], "mathemat": [14, 18, 70, 71], "w_": 14, "b_": 14, "w": [14, 67], "layernormpr": [14, 26, 29], "eff": 14, "ext": 14, "wise": [14, 36], "computation": [14, 71], "wish": 14, "defin": [14, 18, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 69, 70, 71], "x_1": [14, 71], "x_0": [14, 71], "x_2": [14, 71], "frac": [14, 71], "x_3": 14, "cdot": 14, "x_4": 14, "preced": [14, 69, 70, 71], "never": [14, 71], "w_write": 14, "keepdim": 14, "fed": [14, 36], "1000": [14, 35, 69, 71], "recreat": 14, "onto": [14, 38, 70], "By": [14, 35, 37, 38, 40, 69, 70, 71], "mix": [14, 69, 70, 71], "linearli": 14, "technic": [14, 70, 71], "deriv": [14, 71], "broadcast_b_v": 14, "broadcast": 14, "And": [14, 40, 70, 71], "destination_posit": [14, 71], "source_posit": [14, 71], "source_": 14, "destin": [14, 15, 40, 71], "behavior": [14, 15, 38, 70], "cache_dir": [14, 69], "torch_dtyp": 14, "bfloat16": 14, "boolean": [14, 37, 40, 69, 70, 71], "max_new_token": [14, 71], "stop_at_eo": 14, "eos_token_id": [14, 69], "do_sampl": 14, "top_k": [14, 69, 70, 71], "top_p": [14, 69], "temperatur": [14, 69, 71], "freq_penalti": [14, 69], "use_past_kv_cach": 14, "verbos": 14, "pos_plus_new_token": 14, "eos_token": 14, "reach": [14, 71], "avoid": [14, 15, 39, 69, 70, 71], "fiddl": 14, "rag": 14, "eot": 14, "throw": 14, "enter": [14, 70, 71, 73], "messi": [14, 71], "maximum": [14, 15, 18, 63, 71], "stable_lm": 14, "distribut": [14, 68, 69, 70, 71], "greedi": [14, 69], "search": [14, 36, 70, 71], "max": [14, 70], "mass": 14, "random": [14, 15, 35, 63, 70, 71], "temp": [14, 69], "inf": 14, "uniform": [14, 69], "frequenc": [14, 69, 70], "penalti": [14, 69], "penalis": 14, "speed": [14, 70], "applic": [14, 15, 69], "whatev": [14, 70], "tqdm": [14, 71], "get_token_posit": [14, 70, 71], "single_token": [14, 71], "present": 14, "gotcha": [14, 16, 70], "Be": 14, "care": [14, 20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 70, 71], "weird": [14, 15, 70, 71], "carefulli": [14, 70], "correspond": [14, 24, 36, 40, 69, 70, 71], "dummi": [14, 37, 71], "init_weight": [14, 15], "empti": [14, 37], "bulk": 14, "seed": [14, 15, 63, 71], "determin": [14, 18, 40, 68, 69, 70, 71], "NOT": [14, 37, 69, 71], "scheme": 14, "tell": [14, 35, 70, 71], "round": [14, 35, 70, 71], "18182": 14, "fan_in": [14, 69], "tha": 14, "kaim": [14, 69], "despit": [14, 71], "xavier": [14, 69], "fan_out": 14, "transformerencod": 14, "exact": 14, "72253": 14, "mup": [14, 15], "haven": 14, "2203": 14, "03466": 14, "input_to_emb": 14, "special": [14, 71], "redwood": [14, 70, 71], "load_sample_training_dataset": 14, "dataset": [14, 35, 63, 69, 71], "10k": [14, 35, 69], "get_dataset": [14, 69], "appropri": [14, 71], "info": [14, 15, 40, 69, 71], "download": [14, 69, 71], "locat": [14, 40, 70], "pt": 14, "openwebtext": [14, 35, 69], "karma": [14, 35], "reddit": [14, 35], "pile": [14, 35, 38, 69, 71, 72], "imperfectli": 14, "suppli": 14, "valid": [14, 35, 70], "loss_fn": [14, 71], "per_token": [14, 69, 71], "lm_cross_entropy_loss": [14, 69], "move_model_modules_to_devic": 14, "process_weights_": 14, "cleaner": 14, "experiment": 14, "argu": [14, 71], "somewhat": [14, 70, 71], "w_qk": [14, 18, 71], "w_ov": [14, 18, 71], "hopefulli": [14, 73], "column": [14, 40, 69], "rotat": [14, 15, 18, 71], "nth": 14, "formula": 14, "r": 14, "refactor": 14, "diagon": [14, 70, 71], "asymmetri": 14, "fiddli": 14, "preserv": [14, 70, 71], "too": [14, 40, 70], "bilinear": [14, 71], "dimension": [14, 15], "coordin": 14, "sample_datapoint": 14, "implicitli": [14, 40, 71], "hasn": 14, "manual": [14, 69, 71], "replac": [14, 15, 40, 70, 71, 73], "choic": [14, 70], "truncat": [14, 35, 69, 71], "set_token": [14, 15], "pretrainedtoken": 14, "set_use_attn_in": 14, "use_attn_in": [14, 15], "set_use_attn_result": 14, "easili": [14, 69, 70, 71], "burn": 14, "set_use_hook_mlp_in": 14, "use_hook_mlp_in": [14, 15], "set_use_split_qkv_input": 14, "use_split_qkv_input": [14, 15], "to_single_str_token": 14, "int_token": 14, "uncertain": 14, "to_token": [14, 69, 70, 71], "to_str_token": [14, 16, 70, 71], "weirdli": [14, 70, 71], "gotcha2": 14, "letter": [14, 71], "capit": [14, 70, 71], "shoot": [14, 71], "gotcha3": 14, "exce": 14, "str_token": [14, 70], "to_str": [14, 70, 71], "numpi": [14, 15, 69, 70], "arrai": [14, 16, 69], "long": [14, 71], "window": [14, 15, 69], "tokens_to_residual_direct": [14, 70], "mislead": [14, 70], "integ": [14, 69, 70, 71], "residual_direct": 14, "namedtupl": 14, "dataclass": [15, 37], "act_fn": [15, 72], "ep": 15, "1e": [15, 38], "05": [15, 38], "use_attn_scal": 15, "attn_scal": 15, "use_local_attn": 15, "original_architectur": 15, "from_checkpoint": 15, "checkpoint_label_typ": [15, 71], "window_s": [15, 18, 19, 24], "attn_typ": [15, 18, 19, 24, 30], "init_mod": 15, "normalization_typ": 15, "attention_dir": 15, "attn_onli": [15, 72], "initializer_rang": 15, "scale_attn_by_inverse_layer_idx": 15, "final_rm": 15, "d_vocab_out": [15, 34], "parallel_attn_mlp": 15, "rotary_dim": [15, 18], "n_param": [15, 72], "use_hook_token": 15, "gated_mlp": 15, "tokenizer_prepends_bo": 15, "n_key_value_head": [15, 24, 72], "post_embedding_ln": 15, "rotary_bas": 15, "10000": [15, 18, 71], "trust_remote_cod": 15, "rotary_adjacent_pair": 15, "load_in_4bit": 15, "num_expert": 15, "experts_per_token": 15, "relative_attention_max_dist": 15, "relative_attention_num_bucket": 15, "decoder_start_token_id": 15, "tie_word_embed": 15, "use_normalization_before_and_aft": 15, "attn_scores_soft_cap": 15, "output_logits_soft_cap": 15, "AND": 15, "feedforward": 15, "network": [15, 70, 71], "vocab": 15, "lowercas": 15, "relu": [15, 69, 72], "gelu": [15, 38, 71, 72], "silu": [15, 72], "gelu_new": [15, 69], "gelu_fast": [15, 69], "epsilon": 15, "5": [15, 18, 35, 36, 38, 40, 69, 70, 71, 72], "THEN": 15, "intens": 15, "divid": [15, 36, 69, 70], "distanc": [15, 18, 70], "xavier_uniform": 15, "xavier_norm": 15, "kaiming_uniform": 15, "kaiming_norm": 15, "rmspre": 15, "pipelin": 15, "parallel": [15, 69, 70], "aka": 15, "unidirect": 15, "bidirect": [15, 71], "8": [15, 18, 35, 36, 70, 71, 72], "gain": [15, 69], "layer_id": [15, 18, 19, 24, 30], "numer": [15, 16, 18, 19, 24, 71], "stabil": [15, 18, 19, 24, 71], "fp16": 15, "rotari": [15, 18], "blog": [15, 18], "eleuth": [15, 18, 69, 71], "res_stream": 15, "sinusoid": 15, "dumb": 15, "mainli": 15, "curs": 15, "init": 15, "law": 15, "pdf": [15, 18, 32, 35], "2001": 15, "08361": 15, "Will": [15, 40], "interven": [15, 37, 40, 70], "add_bos_token": [15, 69], "bitsandbyt": 15, "moe": 15, "bucket": 15, "tie": 15, "word": [15, 69, 70, 71], "gemma": [15, 38, 42, 72], "softcap": 15, "soft_cap": 15, "tanh": 15, "squash": 15, "interv": [15, 36], "from_dict": 15, "config_dict": 15, "is_layer_norm_activ": 15, "set_seed_everywher": 15, "to_dict": 15, "unwrap": [15, 69], "duplic": [15, 36, 70, 71], "get_singular_vector": 16, "vector_typ": 16, "layer_index": [16, 70], "num_vector": 16, "plot": [16, 71], "pysvelt": [16, 71], "instabl": 16, "d": [16, 35, 36, 38, 70, 72], "medium": [16, 38, 72], "svd_interpret": 16, "22": [16, 35, 69, 70, 71], "all_token": 16, "np": [16, 69, 70], "def": [16, 70, 71], "plot_matrix": 16, "filter": [16, 37, 38, 69, 71], "topk": [16, 70], "topktabl": 16, "obj_typ": 16, "abstract_attent": 17, "bert_block": 17, "bert_emb": 17, "bert_mlm_head": 17, "grouped_query_attent": 17, "layer_norm": [17, 70], "layer_norm_pr": 17, "rms_norm": 17, "rms_norm_pr": 17, "t5_attent": 17, "t5_block": 17, "token_typed_emb": 17, "transformer_block": 17, "abstractattent": [18, 19, 24, 30], "abc": [18, 71], "pure": 18, "glossari": 18, "sorri": 18, "underli": [18, 40, 70, 71], "destination_residu": 18, "destination_po": 18, "source_po": [18, 71], "abstract": [18, 70, 71], "groupedqueryattent": [18, 24], "enforc": 18, "child": 18, "better_abc": 18, "abstract_attribut": 18, "stackoverflow": 18, "question": [18, 70, 71], "23831510": 18, "256": [18, 19, 24, 71, 72], "alibi": 18, "apply_causal_mask": 18, "pos_plus_past_kv_pos_offset": 18, "past_kv_pos_offset": [18, 27, 69], "offset_po": [18, 27, 33, 69], "apply_rotari": 18, "calculate_attention_scor": [18, 24], "calculate_qkv_matric": [18, 24], "query_input": [18, 24], "key_input": [18, 24], "kv_po": [18, 30, 31], "value_input": [18, 24], "calculate_sin_cos_rotari": 18, "sine": 18, "cosin": 18, "wave": 18, "inexplic": 18, "adjac": [18, 70], "neox": [18, 38, 42, 71, 72], "clue": [18, 70], "resolv": 18, "calculate_z_scor": [18, 24], "static": [18, 35], "create_alibi_bia": 18, "head_idx": 18, "2108": 18, "12409": 18, "broad": [18, 70], "proport": [18, 69], "distant": 18, "0000": [18, 70], "0625": 18, "1250": 18, "1875": 18, "0039": 18, "0078": 18, "0117": 18, "create_alibi_multipli": 18, "geometr": 18, "ratio": [18, 69, 70, 71], "16": [18, 69, 70, 71, 72], "5000": 18, "2500": [18, 70], "0312": 18, "0156": 18, "7071": 18, "3536": 18, "1768": 18, "0884": 18, "0442": 18, "0221": 18, "0110": 18, "0055": 18, "create_alibi_slop": 18, "slope": 18, "triangl": 18, "lower": [18, 35, 36, 69, 70, 71], "bottom": [18, 71], "corner": 18, "kv_head_index": [18, 24], "past_kv_cache_entri": [18, 31, 33], "hookedtransformerkeyvaluecacheentri": [18, 31, 33, 39], "additive_attention_mask": [18, 20, 31], "position_bia": [18, 30, 31], "irrelev": [18, 70, 71], "past": [18, 39, 70], "rotate_every_two": 18, "x0": 18, "x1": 18, "param": [19, 63, 69, 71], "convent": [19, 69, 70, 71], "mistal": [19, 24], "bertblock": 20, "transformerblock": [20, 33], "except": [20, 70, 71], "overridden": [20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 69], "subclass": [20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37], "recip": [20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37], "afterward": [20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37], "former": [20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37], "regist": [20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37], "latter": [20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37, 71], "silent": [20, 21, 22, 23, 25, 26, 28, 29, 32, 34, 37], "bertemb": 21, "input_id": 21, "bertmlmhead": 22, "purpos": [22, 35, 70, 71], "resid": 22, "2305": 24, "13245": 24, "hood": 24, "_w_k": 24, "_w_v": 24, "getter": 24, "similarli": 24, "kept": 24, "repeat_interleav": 24, "unexpand": 24, "expan": 24, "n_query_head": 24, "gpa": 24, "normalis": [26, 70], "posemb": 27, "root": [28, 29, 71], "rmsnormpr": 29, "t5attent": [30, 31], "has_relative_attention_bia": 30, "expect": [30, 35, 36, 70, 71], "precomput": 30, "feed": [30, 35, 69, 71], "positional_bia": 30, "compute_relative_attention_bia": 30, "query_len": 30, "key_len": 30, "query_length": 30, "key_length": 30, "bin": 30, "t5block": 31, "block_index": [31, 33], "is_decod": 31, "t5layernorm": 31, "inst": 31, "encoder_additive_attention_mask": 31, "encoder_po": 31, "encoder_hidden_st": 31, "_description_": 31, "_type_": [31, 37], "tokentypeemb": 32, "1810": 32, "04805": 32, "apply_mlp": 33, "normalized_resid": 33, "central": [33, 70], "positional_embeddings_typ": 33, "canbeusedasmlp": 33, "evalu": [35, 37, 70, 71], "rough": [35, 71], "cheapli": 35, "roughli": [35, 70, 71], "baselin": 35, "ioidataset": 35, "noun": 35, "num_sampl": 35, "symmetr": 35, "ioi_ev": 35, "476": 35, "met": 35, "alic": 35, "bob": 35, "charli": 35, "ball": [35, 70], "book": 35, "397": 35, "get_default_nam": 35, "get_default_noun": 35, "get_default_templ": 35, "get_sampl": 35, "evaluate_on_dataset": 35, "data_load": 35, "induction_loss": [35, 71], "subseq_len": 35, "384": [35, 71], "io": [35, 36, 70, 71], "accuraci": [35, 36, 69], "make_code_data_load": 35, "codeparrot": [35, 69], "dump": 35, "presum": [35, 70], "natur": [35, 70, 71], "make_owt_data_load": 35, "corpu": [35, 69], "make_pile_data_load": 35, "eleutherai": [35, 38], "english": [35, 71, 73], "academ": 35, "internet": [35, 71], "make_wiki_data_load": 35, "wikitext": 35, "wikipedia": [35, 69, 71], "articl": [35, 69, 70, 71], "bother": 35, "quarantin": 35, "nowadai": 35, "leakag": 35, "though": [35, 69, 70, 71], "sanity_check": 35, "paragraph": [35, 71], "zoom": [35, 40, 70], "quick": [35, 36, 71], "saniti": [35, 70], "ok": [35, 70, 71], "gone": [35, 70, 71], "wrong": [35, 37, 70], "compute_head_attention_similarity_scor": 36, "attention_pattern": [36, 71], "detection_pattern": 36, "exclude_bo": 36, "exclude_current_token": 36, "error_measur": 36, "mul": 36, "exclude_bcurrent_token": 36, "detect_head": 36, "seq": [36, 69], "previous_token_head": 36, "duplicate_token_head": 36, "induction_head": 36, "headnam": 36, "straightforward": [36, 70], "fraction": 36, "alloc": 36, "prohibit": 36, "cours": [36, 70], "raw": [36, 70], "perfect": [36, 70], "examin": 36, "switch": 36, "advantag": 36, "closer": 36, "head_nam": 36, "ntensor": 36, "ioi": [36, 70, 71], "spacifi": 36, "analyz": 36, "paid": [36, 70, 71], "get_duplicate_token_head_detection_pattern": 36, "dynalist": 36, "n2zwtnoyhru1s4vnfsaq519j": 36, "2ukvedzonghl5uhugvhroxeo": 36, "get_induction_head_detection_pattern": 36, "_tfvup5csv5orithmqwj0gsi": 36, "get_previous_token_head_detection_pattern": 36, "0o5vohe9xezn8ertywkh7ioc": 36, "get_supported_head": 36, "hookfunct": 37, "_hookfunctionprotocol": 37, "inspir": [37, 71, 73], "garcon": [37, 71, 73], "ident": [37, 69, 70, 71], "wrap": [37, 71], "add_hook": [37, 70], "bwd": 37, "fn": 37, "hook_nam": 37, "add_perma_hook": [37, 71], "clear_context": 37, "remove_hook": 37, "including_perman": 37, "interfac": [37, 71, 73], "nice": [37, 70], "run_with_hook": [37, 70, 71], "temporari": [37, 69, 71], "debug": [37, 38, 63], "intent": 37, "reset_hook": [37, 71], "goe": [37, 70, 71], "reset_hooks_end": [37, 70], "add_caching_hook": 37, "names_filt": [37, 70], "callabl": [37, 40], "incl_bwd": 37, "namesfilt": 37, "lambda": [37, 70, 71], "cache_som": 37, "check_and_add_hook": 37, "get_caching_hook": 37, "fwd_hook": [37, 70, 71], "bwd_hook": 37, "hook_dict": 37, "exit": [37, 69], "clear": [37, 71], "reset": 37, "my_hook": 37, "hooked_loss": 37, "mod_dict": 37, "remove_all_hook_fn": 37, "model_kwarg": 37, "degrad": 37, "lenshandl": 37, "removablehandl": 37, "context_level": 37, "hold": 37, "perman": 37, "hug": 38, "face": 38, "hub": [38, 69], "768": [38, 70, 71, 72], "layer_norm_ep": 38, "init_rang": 38, "02": 38, "1024": [38, 69, 71, 72], "3072": [38, 71, 72], "12": [38, 70, 71, 72], "model_alias": 38, "01": 38, "yi": [38, 72], "34b": [38, 72], "6b": [38, 71, 72], "arthurconmi": 38, "redwood_attn_2l": [38, 72], "baidicoot": 38, "codellama": [38, 72], "hf": 38, "codellamallama": [38, 72], "3b": [38, 71, 72], "125m": [38, 71, 72], "20b": [38, 71, 72], "pythia": [38, 72], "4b": [38, 72], "dedup": [38, 72], "12b": [38, 72], "13b": [38, 71, 72], "14m": [38, 72], "160m": [38, 72], "seed1": [38, 72], "seed2": [38, 72], "seed3": [38, 72], "1b": [38, 72], "800m": 38, "8b": [38, 71, 72], "31m": [38, 72], "410m": [38, 72], "350m": 38, "9b": [38, 72], "70m": [38, 72], "19m": [38, 72], "2l512w": 38, "lr": [38, 63], "attn_only_1l512w_c4_cod": 38, "c4": [38, 69, 71], "attn_only_2l512w_c4_cod": 38, "attn_only_3l512w_c4_cod": 38, "attn_only_4l512w_c4_cod": 38, "gelu_1l512w_c4_cod": 38, "gelu_2l512w_c4_cod": 38, "gelu_3l512w_c4_cod": 38, "gelu_4l512w_c4_cod": 38, "solu_10l1280w_c4_cod": 38, "10l": [38, 71, 72], "solu_10l_v22_old": 38, "solu_12l1536w_c4_cod": 38, "12l": [38, 71, 72], "solu_12l_v23_old": 38, "solu_1l512w_c4_cod": 38, "solu_1l512w_wiki_finetun": 38, "wiki": [38, 69, 70, 71, 72], "finetun": 38, "solu_1l_v9_old": 38, "solu_2l512w_c4_cod": 38, "solu_2l_v10_old": 38, "solu_3l512w_c4_cod": 38, "solu_4l512w_c4_cod": 38, "solu_4l512w_wiki_finetun": 38, "solu_4l_v11_old": 38, "solu_6l768w_c4_cod": 38, "6l": [38, 71, 72], "solu_6l_v13_old": 38, "solu_8l1024w_c4_cod": 38, "8l": [38, 71, 72], "solu_8l_v21_old": 38, "qwen": [38, 42, 61, 72], "14b": [38, 72], "1_8b": 38, "qwen1": [38, 72], "5b": [38, 71, 72], "forev": 38, "mgpt": [38, 72], "bigcod": 38, "santacod": [38, 72], "bigscienc": 38, "1b1": [38, 72], "1b7": [38, 72], "560m": [38, 72], "7b1": [38, 72], "distilgpt2": [38, 71], "distillgpt2": [38, 72], "distil": [38, 71], "facebook": 38, "xxl": 38, "30b": [38, 71, 72], "xxxl": 38, "xl": [38, 71, 72], "66b": [38, 71, 72], "xxxxl": 38, "27b": [38, 72], "2b": [38, 72], "65b": [38, 72], "70b": [38, 72], "microsoft": 38, "phi": [38, 42, 59, 72], "mini": 38, "4k": 38, "1_5": [38, 72], "roneneldan": 38, "tinystori": 38, "1layer": 38, "21m": [38, 72], "28m": [38, 72], "2layer": 38, "33m": [38, 72], "3m": [38, 72], "8m": [38, 72], "instuct": 38, "stabilityai": 38, "stablelm": [38, 71, 72], "alpha": [38, 72], "x21": 38, "arwen": 38, "battlestar": 38, "x49": 38, "beren": 38, "caprica": 38, "x81": 38, "celebrimbor": 38, "darkmatt": 38, "x343": 38, "durin": 38, "eowyn": 38, "x777": 38, "expans": 38, "alias": 38, "non_hf_hosted_model_nam": 38, "qwen2": [38, 42, 72], "get_checkpoint_label": [38, 71], "label_typ": 38, "get_num_params_of_pretrain": 38, "suffici": [38, 70], "get_pretrained_model_config": 38, "hf_cfg": 38, "automodel": 38, "autoconfig": 38, "infrastructur": [38, 70, 71, 73], "ourselv": [39, 69, 71, 73], "previous_attention_mask": 39, "pos_so_far": 39, "append": [39, 70, 71], "prefix": 39, "append_attention_mask": 39, "new_token": 39, "freez": 39, "init_cach": 39, "unfreez": 39, "past_kei": 39, "jaxtyp": [39, 70, 71], "past_valu": 39, "new_kei": 39, "new_valu": 39, "init_cache_entri": 39, "structur": [40, 71], "generic_activation_patch": 40, "specialis": [40, 70], "introduc": [40, 70], "rome": [40, 70, 71], "baulab": 40, "corrupt": [40, 70, 71], "iter": [40, 69, 70, 71], "localis": [40, 70, 71], "__from__": 40, "__to": 40, "__the": 40, "confid": [40, 70, 71], "intuit": [40, 70, 71], "diffus": [40, 70], "spread": [40, 70], "connect": [40, 70], "ultim": [40, 70], "tend": [40, 71], "extrem": [40, 70, 71, 73], "eiffel": 40, "tower": 40, "pari": 40, "factual": [40, 70], "recal": [40, 70], "colosseum": 40, "corrupted_token": [40, 70, 71], "clean_cach": [40, 70, 71], "patching_metr": 40, "patch_sett": 40, "index_axis_nam": 40, "src_po": [40, 70], "dest_po": [40, 70, 71], "index_df": 40, "datafram": 40, "return_index_df": 40, "counterfactu": [40, 70, 71], "Then": 40, "index_to_act_nam": 40, "recov": [40, 70, 71], "diff": [40, 70], "corrupted_activ": 40, "chunk": 40, "fill": 40, "flatten": [40, 70, 71], "patched_output": 40, "get_act_patch_attn_head_all_pos_everi": 40, "patch_typ": 40, "get_act_patch_attn_head_by_pos_everi": 40, "get_act_patch_attn_head_k_all_po": 40, "corruptedactiv": 40, "patchedactiv": 40, "layer_head_vector_patch_sett": 40, "axisnam": 40, "get_act_patch_attn_head_k_by_po": 40, "layer_pos_head_vector_patch_sett": 40, "get_act_patch_attn_head_out_all_po": 40, "get_act_patch_attn_head_out_by_po": 40, "get_act_patch_attn_head_pattern_all_po": 40, "layer_head_pattern_patch_sett": 40, "get_act_patch_attn_head_pattern_by_po": 40, "layer_head_pos_pattern_patch_sett": 40, "get_act_patch_attn_head_pattern_dest_src_po": 40, "layer_head_dest_src_pos_pattern_patch_sett": 40, "get_act_patch_attn_head_q_all_po": 40, "get_act_patch_attn_head_q_by_po": 40, "get_act_patch_attn_head_v_all_po": 40, "get_act_patch_attn_head_v_by_po": 40, "get_act_patch_attn_out": 40, "layer_pos_patch_sett": 40, "get_act_patch_block_everi": 40, "get_act_patch_mlp_out": 40, "get_act_patch_resid_mid": 40, "get_act_patch_resid_pr": 40, "clean_activ": 40, "weight_convers": 41, "coder": 42, "mingpt": 42, "nanogpt": 42, "neel_solu_old": 42, "phi3": 42, "convert_bloom_weight": 44, "convert_coder_weight": 45, "convert_mistral_weight": 51, "convert_mixtral_weight": 52, "convert_phi_weight": 58, "convert_phi3_weight": 59, "convert_qwen_weight": 60, "convert_qwen2_weight": 61, "convert_t5_weight": 62, "hookedtransformertrainconfig": 63, "num_epoch": 63, "001": 63, "max_grad_norm": 63, "weight_decai": 63, "optimizer_nam": 63, "adam": 63, "warmup_step": 63, "save_everi": 63, "save_dir": 63, "wandb": 63, "wandb_project_nam": 63, "print_everi": 63, "max_step": 63, "hyperparamet": [63, 69], "epoch": 63, "rate": [63, 71], "decai": 63, "warmup": 63, "wandb_project": 63, "termin": 63, "activation_funct": 64, "addmm": 64, "batch_addmm": 66, "d_out": [66, 69], "d_in": [66, 69], "fuse": 66, "conv1d": 66, "9ba9369a2557e53a01378199a9839ec6e82d8bc7": 66, "src": 66, "pytorch_util": 66, "l102": 66, "l106": 66, "vanilla_addmm": 66, "mat1": 66, "mat2": 66, "typecheck": 66, "complex_attn_linear": 67, "almost": [67, 70], "simple_attn_linear": 67, "extra": [67, 69], "assist": 68, "get_device_for_block_index": 68, "target": 68, "move_to_and_update_config": 68, "vari": [69, 70], "throughout": [69, 71], "locallyoverridendefault": 69, "restor": 69, "overriden": 69, "input_slic": 69, "syntax": [69, 70, 71], "reduc": [69, 70, 71], "leav": [69, 71], "elif": 69, "1d": 69, "sliceinput": 69, "valueerror": 69, "abov": [69, 70, 71], "max_ctx": 69, "int32": 69, "slice_input": 69, "calc_fan_in_and_fan_out": 69, "fan": 69, "composition_scor": 69, "broadcast_dim": 69, "leading_dims_left_and_right": 69, "download_file_from_hf": 69, "repo_nam": 69, "file_nam": 69, "subfold": 69, "home": 69, "runner": 69, "force_is_torch": 69, "json": 69, "pth": 69, "extens": [69, 70], "layer_typ": [69, 70], "shorthand": 69, "loop": [69, 70, 71, 73], "hack": [69, 71], "stuff": [69, 71], "readabl": 69, "digit": [69, 71], "k6": 69, "scale4ln1": 69, "appear": [69, 71], "distinguish": [69, 70], "hook_k": [69, 71], "hook_pr": [69, 71], "27": [69, 70, 71], "hook_norm": [69, 71], "pre5": 69, "get_attention_mask": 69, "leftmost": 69, "rightmost": 69, "consid": 69, "get_cumsum_along_dim": 69, "dataset_nam": 69, "000": [69, 71], "enorm": [69, 71], "100gb": 69, "2tb": 69, "effort": [69, 70], "dataload": 69, "fanci": 69, "data_dir": 69, "approx": [69, 70, 71], "ton": [69, 73], "divers": [69, 70, 71], "coloss": 69, "crawl": 69, "bigger": 69, "c4_code": 69, "friendli": 69, "22m": [69, 71], "5m": 69, "20220301": 69, "en": [69, 71], "get_devic": [69, 70, 71], "get_input_with_manually_prepended_bo": 69, "autotoken": 69, "get_nested_attr": 69, "obj": 69, "attr_str": 69, "retriev": 69, "nest": 69, "hierarchi": 69, "get_offset_position_id": 69, "offset": [69, 70, 71], "get_tokenizer_with_bo": 69, "Such": [69, 70], "llamatoken": 69, "get_tokens_with_bos_remov": 69, "init_kaiming_normal_": 69, "nonlinear": 69, "std": 69, "init_kaiming_uniform_": 69, "init_xavier_normal_": 69, "init_xavier_uniform_": 69, "is_lower_triangular": 69, "is_squar": 69, "keep_single_column": 69, "col_nam": 69, "lm_accuraci": 69, "seq_len": [69, 70, 71], "altern": 69, "override_or_use_default_valu": 69, "default_flag": 69, "print_gpu_mem": 69, "step_nam": 69, "repeat_along_head_dimens": 69, "clone_tensor": 69, "sample_logit": 69, "final_logit": [69, 70], "vocab_s": 69, "high": [69, 70, 71], "argmaxi": 69, "90": 69, "renormalis": 69, "mutual": 69, "neither": [69, 70], "input_token": 69, "todo": 69, "edg": 69, "randn": [69, 71], "uniqu": 69, "return_count": 69, "set_nested_attr": 69, "prepend_space_to_answ": 69, "eleph": 69, "endoftext": [69, 70, 71], "14": [69, 70, 71, 72], "51": [69, 71], "0th": [69, 70], "59": [69, 71, 72], "ground": [69, 70], "1th": [69, 70], "41": [69, 71], "tree": 69, "2th": [69, 70], "3th": [69, 70], "45": [69, 71], "car": 69, "4th": [69, 70], "13": [69, 70, 71], "92": [69, 70], "55": [69, 70, 71], "river": 69, "5th": [69, 70], "79": 69, "25": [69, 70, 71, 72], "street": 69, "6th": [69, 70], "77": 69, "21": [69, 70, 71], "7th": [69, 70], "75": 69, "hill": 69, "8th": [69, 70], "swing": 69, "9th": [69, 70], "46": [69, 71, 72], "61": [69, 72], "park": [69, 70], "to_numpi": [69, 70, 71], "tokenize_and_concaten": 69, "max_length": 69, "column_nam": 69, "num_proc": 69, "eo": [69, 71], "reshap": [69, 70], "____": 69, "drop": [69, 71], "faster": [69, 70, 71], "parallelis": [69, 71], "chop": 69, "privileg": 69, "earli": [69, 71], "cnn": [69, 71], "bos_token_id": 69, "swap": [69, 70], "runtim": [70, 71], "hardwar": [70, 71], "pane": [70, 71], "sidebar": [70, 71], "navig": [70, 71], "vscode": [70, 71], "outlin": 70, "tab": 70, "dropdown": [70, 71], "arrow": [70, 71], "page": [70, 71], "ctrl": [70, 71], "in_colab": [70, 71], "circuitsvi": [70, 71], "node": [70, 71], "curl": [70, 71], "fssl": [70, 71], "deb": [70, 71], "nodesourc": [70, 71], "setup_16": [70, 71], "sudo": [70, 71], "bash": [70, 71], "apt": [70, 71], "nodej": [70, 71], "noqa": [70, 71], "ipython": [70, 71], "get_ipython": [70, 71], "ip": [70, 71], "extension_manag": [70, 71], "autoreload": [70, 71], "functool": [70, 71], "plotli": [70, 71], "express": [70, 71], "px": [70, 71], "pio": [70, 71], "attention_head": 70, "fancy_einsum": [70, 71], "ifram": 70, "differenti": [70, 71], "simplic": 70, "imshow": [70, 71], "color_continuous_midpoint": [70, 71], "color_continuous_scal": [70, 71], "rdbu": [70, 71], "scatter": [70, 71], "xaxi": [70, 71], "yaxi": [70, 71], "caxi": [70, 71], "color": [70, 71], "principl": [70, 71, 73], "fun": [70, 71, 73], "gap": [70, 71, 73], "plai": [70, 71, 73], "flow": [70, 71, 73], "toolkit": [70, 71], "stylist": 70, "slowli": 70, "convei": 70, "tag": 70, "asid": 70, "flavour": 70, "weed": 70, "star": 70, "tagexampl": 70, "capabl": [70, 71], "interview": [70, 71], "kevin": [70, 71], "wang": 70, "twitter": 70, "thread": 70, "overview": 70, "bottl": [70, 71], "milk": [70, 71], "26": [70, 71, 72], "Their": 70, "skimp": 70, "rigour": 70, "suggest": 70, "evid": 70, "80m": [70, 71], "simplif": 70, "nbval_ignore_output": [70, 71], "stabl": 70, "example_prompt": 70, "example_answ": 70, "39": [70, 71], "lt": [70, 71], "gt": [70, 71], "09": [70, 71], "70": 70, "07": [70, 71], "38": [70, 71], "67": 70, "35": [70, 71], "54": [70, 71], "11": [70, 71, 72], "84": [70, 71], "73": 70, "hi": [70, 71], "06": 70, "her": [70, 71], "74": 70, "52": [70, 71, 72], "49": [70, 71], "jesu": 70, "97": 70, "42": [70, 71, 72], "him": 70, "subword": 70, "frequent": 70, "substr": [70, 71], "headach": 70, "annoi": [70, 71], "devot": 70, "sensibl": 70, "later": [70, 71], "wherev": 70, "flesh": 70, "prompt_format": 70, "jame": 70, "dan": 70, "sid": 70, "appl": 70, "martin": 70, "ami": 70, "drink": 70, "correct_token": 70, "insert": 70, "filler": 70, "newlin": 70, "intellig": 70, "complic": 70, "adjust": [70, 71], "aggreg": 70, "original_logit": 70, "upon": 70, "subject": [70, 71], "logits_to_ave_logit_diff": 70, "per_prompt": 70, "answer_logit": 70, "gather": 70, "answer_logit_diff": 70, "detach": [70, 71], "decim": [70, 71], "original_average_logit_diff": 70, "3370": 70, "2020": 70, "7090": 70, "7970": 70, "7200": 70, "2810": 70, "6010": 70, "7670": 70, "552": 70, "33": [70, 71], "dive": 70, "spend": [70, 71], "engag": 70, "decent": [70, 71], "hypothes": 70, "cheat": [70, 71], "hypothesi": 70, "scienc": 70, "belief": 70, "trap": 70, "flounder": 70, "dogmat": 70, "overconfid": 70, "unwil": 70, "contradict": 70, "flinch": 70, "disconfirm": 70, "focu": 70, "primit": 70, "nearbi": 70, "came": 70, "trigram": 70, "symmetri": 70, "cancel": 70, "inhibit": 70, "spoiler": 70, "simplist": 70, "importantli": [70, 71], "perfectli": [70, 71], "final_residual_stream": 70, "eleg": 70, "particularli": 70, "aspect": 70, "nicer": 70, "inde": 70, "log_prob": 70, "log_softmax": 70, "logsumexp": 70, "decid": 70, "pronoun": 70, "refin": 70, "friendlier": 70, "answer_residual_direct": 70, "logit_diff_direct": 70, "account": 70, "w_u_fold": 70, "unigram": [70, 71], "statist": [70, 71], "opposit": 70, "hook_normalis": 70, "sub_layer_typ": 70, "final_token_residual_stream": 70, "scaled_final_token_residual_stream": 70, "average_logit_diff": 70, "residual_stack_to_logit_diff": 70, "scaled_residual_stack": 70, "fascinatingli": 70, "utterli": 70, "unabl": 70, "hover": [70, 71], "n_pre": 70, "n_mid": 70, "n_post": 70, "middl": [70, 71], "accumulated_residu": 70, "logit_lens_logit_diff": 70, "arang": 70, "hover_nam": [70, 71], "terminologi": 70, "overload": 70, "kth": 70, "per_layer_residu": 70, "per_layer_logit_diff": 70, "independ": [70, 71, 73], "l9h6": 70, "l9h9": 70, "l10h7": 70, "l11h10": 70, "harm": 70, "strongli": 70, "observ": [70, 71], "144": 70, "claim": 70, "surpris": 70, "7x": 70, "per_head_residu": 70, "per_head_logit_diff": 70, "rearrang": 70, "weren": 70, "alan": [70, 71], "coonei": [70, 71], "illustr": [70, 71], "mistak": 70, "mayb": [70, 71], "sai": [70, 71], "summari": 70, "sole": 70, "visualize_attention_pattern": 70, "local_cach": 70, "local_token": 70, "max_width": 70, "700": 70, "isinst": 70, "batch_index": 70, "combin": [70, 71], "attention_head_nam": 70, "show_cod": 70, "title_html": 70, "br": 70, "div": 70, "width": [70, 71], "top_positive_logit_attr_head": 70, "positive_html": 70, "top_negative_logit_attr_head": 70, "negative_html": 70, "conceptu": 70, "clearli": 70, "compos": [70, 71], "ideal": [70, 71], "david": [70, 71], "bau": [70, 71], "meng": [70, 71], "trace": [70, 71], "anim": 70, "lai": 70, "pro": 70, "con": 70, "Or": 70, "bake": 70, "claus": 70, "tack": 70, "gaussian": 70, "nois": 70, "beforehand": 70, "19": [70, 71], "corrupted_prompt": [70, 71], "corrupted_logit": [70, 71], "corrupted_cach": 70, "corrupted_average_logit_diff": 70, "temporarili": [70, 71], "patch_residual_compon": 70, "corrupted_residual_compon": 70, "normalize_patched_logit_diff": 70, "patched_logit_diff": [70, 71], "wors": [70, 71], "patched_residual_stream_diff": 70, "hook_fn": 70, "patched_logit": [70, 71], "abus": 70, "prompt_position_label": 70, "tok": 70, "_": [70, 71], "enumer": [70, 71], "reus": 70, "patched_attn_diff": 70, "patched_mlp_diff": 70, "patched_attn_logit": 70, "patched_attn_logit_diff": 70, "patched_mlp_logit": 70, "patched_mlp_logit_diff": 70, "late": [70, 71], "contrast": 70, "statement": 70, "mlp0": 70, "destroi": 70, "frame": 70, "unprincipl": 70, "invers": [70, 71], "plausibli": 70, "dedic": 70, "overcom": 70, "love": 70, "someon": 70, "patch_head_vector": 70, "corrupted_head_vector": 70, "patched_head_z_diff": 70, "l8h6": 70, "l8h10": 70, "l7h9": 70, "l5h5": 70, "l6h9": 70, "l3h0": 70, "semi": 70, "disentangl": 70, "familiar": 70, "28": [70, 71, 72], "patched_head_v_diff": 70, "heatmap": 70, "29": [70, 71], "lesson": 70, "head_label": 70, "range_x": 70, "range_i": 70, "31": [70, 71], "patch_head_pattern": 70, "corrupted_head_pattern": 70, "patched_head_attn_diff": 70, "32": [70, 71, 72], "reconsolid": 70, "extend": 70, "l7h3": 70, "specul": 70, "mysteri": [70, 71], "top_heads_by_output_patch": 70, "first_mid_lay": 70, "first_late_lay": 70, "early_head": 70, "mid_head": 70, "logical_and": 70, "late_head": 70, "diagram": 70, "l1h2": 70, "latest": 70, "definit": 70, "priori": 70, "stroke": 70, "didn": 70, "bracket": 70, "serv": [70, 71], "particip": 70, "behav": 70, "l5h0": 70, "wrote": [70, 71, 73], "overkil": 70, "simpler": 70, "repurpos": 70, "machineri": 70, "life": [70, 71], "built": 70, "34": [70, 71], "example_text": [70, 71], "seek": 70, "machin": [70, 71], "example_repeated_text": 70, "example_repeated_token": 70, "example_repeated_logit": 70, "example_repeated_cach": 70, "induction_head_label": 70, "81": 70, "65": 70, "800": 70, "accord": 70, "wildli": 70, "characteris": 70, "superfici": 70, "boost": [70, 71], "anti": 70, "suppress": [70, 71], "pick": [70, 71], "signal": 70, "hook_": 70, "hook_attn": 70, "token_po": 70, "metadata": 70, "36": [70, 71, 72], "prev_token_scor": 70, "prev_token_hook": 70, "dim1": [70, 71], "dim2": [70, 71], "duplicate_token_scor": 70, "duplicate_token_hook": 70, "induction_scor": [70, 71], "induction_hook": 70, "manual_se": [70, 71], "original_token": 70, "randint": [70, 71], "20000": [70, 71], "repeated_token": [70, 71], "pattern_filt": 70, "act_nam": [70, 71], "endswith": [70, 71], "hook_pattern": [70, 71], "0390": 70, "0310": 70, "1890": 70, "1720": 70, "0680": 70, "1570": 70, "0210": 70, "4820": 70, "0030": 70, "1320": 70, "0050": 70, "0020": 70, "0090": 70, "0040": 70, "0010": 70, "instantli": 70, "37": [70, 71], "seen": [70, 71], "mosaic": 70, "40": [70, 71, 72], "fascin": 70, "knock": 70, "naiv": [70, 71], "convers": 70, "flaw": 70, "knockout": 70, "send": 70, "redund": 70, "job": 70, "underestim": 70, "57": [70, 71], "99": [70, 71], "hook_z": [70, 71], "top_name_mov": 70, "top_name_mover_lay": 70, "top_name_mover_head": 70, "ablate_top_head_hook": 70, "ablated_logit": 70, "ablated_cach": 70, "2f": [70, 71], "l10h10": 70, "margin": 70, "obvious": 70, "per_head_ablated_residu": 70, "per_head_ablated_logit_diff": 70, "04": [70, 71], "uniformli": [70, 71], "042": 70, "5200": 70, "4700": 70, "8200": 70, "5100": 70, "2600": 70, "1800": 70, "4300": 70, "5700": 70, "3500": 70, "2900": 70, "6800": 70, "4900": 70, "8700": 70, "4200": 70, "reader": [70, 71], "gentler": 71, "tip": 71, "development_mod": 71, "in_github": 71, "getenv": 71, "github_act": 71, "render": 71, "argh": 71, "notebook_connect": 71, "cv": 71, "hello": 71, "auto": 71, "autograd": 71, "grad_mod": 71, "0x7f282d327510": 71, "speak": [71, 73], "human": [71, 73], "palm": [71, 73], "nor": [71, 73], "offend": [71, 73], "anthrop": [71, 73], "team": [71, 73], "got": [71, 73], "frustrat": [71, 73], "deepspe": [71, 73], "industri": [71, 73], "heavili": [71, 73], "credit": [71, 73], "nelson": [71, 73], "elhag": [71, 73], "chri": [71, 73], "olah": [71, 73], "model_description_text": 71, "hyper": 71, "1758": 71, "box": 71, "On": 71, "insid": 71, "kinda": 71, "gpt2_cache_no_batch_dim": 71, "gpt2_cach": 71, "gpt2_text": 71, "summar": 71, "supervis": 71, "taskspecif": 71, "gpt2_token": 71, "gpt2_logit": 71, "lock": 71, "grid": 71, "gpt2_str_token": 71, "neural": 71, "system": 71, "surgic": 71, "surround": 71, "current_activation_valu": 71, "new_activation_valu": 71, "substitut": 71, "relationship": 71, "underr": 71, "janki": 71, "shamelessli": 71, "probepoint": 71, "qualiti": 71, "head_ablation_hook": 71, "layer_to_abl": 71, "head_index_to_abl": 71, "original_loss": 71, "ablated_loss": 71, "3f": 71, "999": 71, "453": 71, "stai": 71, "clean_prompt": 71, "clean_token": 71, "logits_to_logit_diff": 71, "correct_answ": 71, "incorrect_answ": 71, "correct_index": 71, "incorrect_index": 71, "clean_logit": 71, "clean_logit_diff": 71, "corrupted_logit_diff": 71, "276": 71, "738": 71, "residual_stream_patching_hook": 71, "clean_resid_pr": 71, "num_posit": 71, "ioi_patching_result": 71, "temp_hook_fn": 71, "ish": 71, "token_label": 71, "workflow": 71, "michael": 71, "jordan": 71, "surnam": 71, "occurr": 71, "terribl": 71, "halfwai": 71, "input_tensor": 71, "random_token": 71, "repeated_logit": 71, "correct_log_prob": 71, "loss_by_posit": 71, "manipul": 71, "hook_funct": 71, "induction_score_stor": 71, "induction_score_hook": 71, "induction_strip": 71, "pattern_hook_names_filt": 71, "highli": 71, "stripe": 71, "induction_head_lay": 71, "induction_head_index": 71, "single_random_sequ": 71, "repeated_random_sequ": 71, "visualize_pattern_hook": 71, "3d": 71, "four": 71, "300m": 71, "soon": 71, "distilgpt": 71, "distilgpt2_induction_score_stor": 71, "classic": 71, "openai": 71, "85m": [71, 72], "700m": 71, "22b": 71, "300b": 71, "180b": 71, "600": 71, "265": 71, "108m": 71, "bookscorpu": 71, "free": 71, "512": [71, 72], "tractabl": 71, "motif": 71, "80": [71, 72], "shuffl": 71, "scan": 71, "40m": 71, "100m": 71, "200m": 71, "340m": [71, 72], "15b": 71, "13m": [71, 72], "digress": 71, "usefulli": 71, "variengien": 71, "websit": 71, "cleantransformerdemo": 71, "new_activ": 71, "old_activ": 71, "remind": 71, "50267": 71, "named_paramet": 71, "startswith": 71, "fallback": 71, "spam": 71, "dest_posit": 71, "brown": 71, "fox": 71, "lazi": 71, "dog": 71, "num": 71, "print_name_shape_hook_funct": 71, "not_in_late_block_filt": 71, "hook_q": 71, "hook_v": 71, "hook_attn_scor": 71, "hook_attn_out": 71, "hook_resid_mid": 71, "hook_post": 71, "hook_mlp_out": 71, "hook_resid_post": 71, "preconcept": 71, "overhead": 71, "elementwis": 71, "consequ": 71, "rare": 71, "dramat": 71, "degre": 71, "punctuat": 71, "ass": 71, "randomredditor": 71, "unembed_bia": 71, "bias_valu": 71, "bias_indic": 71, "repr": 71, "03": 71, "98": 71, "68": 71, "48": [71, 72], "47": 71, "72": [71, 72], "44": [71, 72], "82": 71, "\u30b5\u30fc\u30c6\u30a3": 71, "83": 71, "x18": 71, "x14": 71, "\u9f8d": 71, "x1b": 71, "x05": 71, "x00": 71, "x06": 71, "x07": 71, "x0c": 71, "x02": 71, "oreandonlin": 71, "x11": 71, "x10": 71, "favour": 71, "6x": 71, "john_bia": 71, "mary_bia": 71, "4f": 71, "exp": 71, "8995": 71, "6034": 71, "6550x": 71, "finit": 71, "invert": 71, "de": 71, "uncommon": 71, "iz": 71, "charact": 71, "example_text_str_token": 71, "example_text_token": 71, "50256": 71, "464": 71, "717": 71, "1517": 71, "345": 71, "761": 71, "284": 71, "3785": 71, "503": 71, "318": 71, "1635": 71, "4919": 71, "1243": 71, "389": 71, "11241": 71, "1143": 71, "4600": 71, "19849": 71, "1462": 71, "62": 71, "2536": 71, "482": 71, "641": 71, "63": 71, "30778": 71, "257": 71, "4731": 71, "656": 71, "262": 71, "16326": 71, "292": 71, "1351": 71, "286": 71, "850": 71, "37336": 71, "25666": 71, "290": 71, "523": 71, "8781": 71, "7301": 71, "644": 71, "2420": 71, "3073": 71, "588": 71, "1675": 71, "10176": 71, "428": 71, "1309": 71, "338": 71, "779": 71, "340": 71, "319": 71, "7322": 71, "signifi": 71, "example_multi_text": 71, "cat": 71, "sat": 71, "mat": 71, "example_multi_text_token": 71, "3797": 71, "3332": 71, "2603": 71, "1107": 71, "1327": 71, "th": 71, "cat_text": 71, "cat_logit": 71, "cat_prob": 71, "capital_the_token_index": 71, "ascii": 71, "squeez": 71, "annoy": 71, "arithmet": 71, "impress": 71, "2342": 71, "2017": 71, "21445": 71, "1000000": 71, "999999": 71, "214": 71, "000000": 71, "9999": 71, "tim": 71, "ne": 71, "el": 71, "messier": 71, "takeawai": 71, "unexpect": 71, "notic": 71, "trip": 71, "confusingli": 71, "forth": 71, "ioi_logits_with_bo": 71, "clair": 71, "mary_logit_with_bo": 71, "claire_logit_with_bo": 71, "ioi_logits_without_bo": 71, "mary_logit_without_bo": 71, "claire_logit_without_bo": 71, "754": 71, "782": 71, "air": 71, "understood": 71, "requisit": 71, "attention_scor": 71, "ab_factor": 71, "9105": 71, "linalg": 71, "eig": 71, "2877e": 71, "00": 71, "8626e": 71, "3121e": 71, "9038e": 71, "08": 71, "1527e": 71, "2877": 71, "3121": 71, "3126e": 71, "3963e": 71, "2029e": 71, "7690e": 71, "2164e": 71, "3126": 71, "3963": 71, "300": 71, "abc_factor": 71, "unfactor": 71, "160": 71, "0830": 71, "43": 71, "ab_unfactor": 71, "isclos": 71, "subspac": 71, "coincid": 71, "assert": 71, "negat": 71, "proxi": 71, "lambda_i": 71, "ov_circuit_all_head": 71, "ov_circuit_all_heads_eigenvalu": 71, "complex64": 71, "ov_copying_scor": 71, "zmax": 71, "zmin": 71, "l11h11": 71, "imag": 71, "imaginari": 71, "full_ov_circuit": 71, "full_ov_circuit_eigenvalu": 71, "full_ov_copying_scor": 71, "interestingli": 71, "correl": 71, "outlier": 71, "ansh": 71, "radhakrishnan": 71, "establish": 71, "53": 71, "presid": 71, "barack": 71, "obama": 71, "caught": 71, "embarrass": 71, "scandal": 71, "nthe": 71, "financi": 71, "wife": 71, "chelsea": 71, "she": 71, "woman": 71, "lightweight": 71, "squarethenadd": 71, "hook_squar": 71, "twolayermodel": 71, "layer1": 71, "layer2": 71, "hook_in": 71, "hook_mid": 71, "hook_out": 71, "x_in": 71, "x_mid": 71, "x_out": 71, "model_out": 71, "cache_object": 71, "780": 71, "784": 71, "56": [71, 72], "set_to_zero_hook": 71, "num_checkpoint": 71, "piecewis": 71, "schedul": 71, "crash": 71, "11b": [71, 72], "centr": 71, "hoc": 71, "count": 71, "checkpoint_label": 71, "log_i": 71, "marker": 71, "brief": 71, "suddenli": 71, "500": 71, "visibl": 71, "curv": 71, "briefli": 71, "deliber": 71, "justic": 71, "chosen": 71, "60": [71, 72], "500m": 71, "58": 71, "arbitrarili": 71, "fast": 71, "checkpoint_indic": 71, "checkpointed_model": 71, "tokens_trained_on": 71, "model_for_this_checkpoint": 71, "tokens_seen_for_this_checkpoint": 71, "induction_loss_for_this_checkpoint": 71, "contextualis": 71, "strategi": 71, "95": 71, "log_x": 71, "302m": 72, "4096": 72, "708m": 72, "1280": 72, "5120": 72, "1600": 72, "6400": 72, "42m": 72, "2048": 72, "50272": 72, "8192": 72, "2560": 72, "10240": 72, "128": 72, "16384": 72, "20480": 72, "7168": 72, "28672": 72, "9216": 72, "36864": 72, "50400": 72, "6144": 72, "50432": 72, "96": 72, "24576": 72, "2m": 72, "50304": 72, "7m": 72, "805m": 72, "50688": 72, "50278": 72, "736": 72, "2944": 72, "101m": 72, "197m": 72, "1536": 72, "48262": 72, "4m": 72, "0m": 72, "50277": 72, "524k": 72, "50259": 72, "32000": 72, "11008": 72, "13824": 72, "32b": 72, "6656": 72, "17920": 72, "22016": 72, "78b": 72, "32016": 72, "128256": 72, "14336": 72, "25m": 72, "28996": 72, "393k": 72, "6m": 72, "47b": 72, "32768": 72, "250880": 72, "679m": 72, "0b": 72, "49280": 72, "151936": 72, "5504": 72, "152064": 72, "13696": 72, "308m": 72, "2816": 72, "6912": 72, "391m": 72, "896": 72, "4864": 72, "8960": 72, "3584": 72, "18944": 72, "51200": 72, "32064": 72, "256000": 72, "2304": 72, "gelu_pytorch_tanh": 72, "4608": 72, "64000": 72, "39b": 72, "32128": 72, "100000": 72, "formerli": 73, "transfer": 73}, "objects": {"transformer_lens": [[10, 0, 0, "-", "ActivationCache"], [11, 0, 0, "-", "FactoredMatrix"], [12, 0, 0, "-", "HookedEncoder"], [13, 0, 0, "-", "HookedEncoderDecoder"], [14, 0, 0, "-", "HookedTransformer"], [15, 0, 0, "-", "HookedTransformerConfig"], [16, 0, 0, "-", "SVDInterpreter"], [35, 0, 0, "-", "evals"], [36, 0, 0, "-", "head_detector"], [37, 0, 0, "-", "hook_points"], [38, 0, 0, "-", "loading_from_pretrained"], [39, 0, 0, "-", "past_key_value_caching"], [40, 0, 0, "-", "patching"], [63, 0, 0, "-", "train"], [69, 0, 0, "-", "utils"]], "transformer_lens.ActivationCache": [[10, 1, 1, "", "ActivationCache"]], "transformer_lens.ActivationCache.ActivationCache": [[10, 2, 1, "", "accumulated_resid"], [10, 2, 1, "", "apply_ln_to_stack"], [10, 2, 1, "", "apply_slice_to_batch_dim"], [10, 2, 1, "", "compute_head_results"], [10, 2, 1, "", "decompose_resid"], [10, 2, 1, "", "get_full_resid_decomposition"], [10, 2, 1, "", "get_neuron_results"], [10, 2, 1, "", "items"], [10, 2, 1, "", "keys"], [10, 2, 1, "", "logit_attrs"], [10, 2, 1, "", "remove_batch_dim"], [10, 2, 1, "", "stack_activation"], [10, 2, 1, "", "stack_head_results"], [10, 2, 1, "", "stack_neuron_results"], [10, 2, 1, "", "to"], [10, 2, 1, "", "toggle_autodiff"], [10, 2, 1, "", "values"]], "transformer_lens.FactoredMatrix": [[11, 1, 1, "", "FactoredMatrix"]], "transformer_lens.FactoredMatrix.FactoredMatrix": [[11, 3, 1, "", "AB"], [11, 3, 1, "", "BA"], [11, 3, 1, "", "S"], [11, 3, 1, "", "T"], [11, 3, 1, "", "U"], [11, 3, 1, "", "Vh"], [11, 2, 1, "", "collapse_l"], [11, 2, 1, "", "collapse_r"], [11, 3, 1, "", "eigenvalues"], [11, 2, 1, "", "get_corner"], [11, 2, 1, "", "make_even"], [11, 3, 1, "", "ndim"], [11, 2, 1, "", "norm"], [11, 3, 1, "", "pair"], [11, 2, 1, "", "svd"], [11, 2, 1, "", "unsqueeze"]], "transformer_lens.HookedEncoder": [[12, 1, 1, "", "HookedEncoder"]], "transformer_lens.HookedEncoder.HookedEncoder": [[12, 3, 1, "", "OV"], [12, 3, 1, "", "QK"], [12, 3, 1, "", "W_E"], [12, 3, 1, "", "W_E_pos"], [12, 3, 1, "", "W_K"], [12, 3, 1, "", "W_O"], [12, 3, 1, "", "W_Q"], [12, 3, 1, "", "W_U"], [12, 3, 1, "", "W_V"], [12, 3, 1, "", "W_in"], [12, 3, 1, "", "W_out"], [12, 3, 1, "", "W_pos"], [12, 2, 1, "", "all_head_labels"], [12, 3, 1, "", "b_K"], [12, 3, 1, "", "b_O"], [12, 3, 1, "", "b_Q"], [12, 3, 1, "", "b_U"], [12, 3, 1, "", "b_V"], [12, 3, 1, "", "b_in"], [12, 3, 1, "", "b_out"], [12, 2, 1, "", "cpu"], [12, 2, 1, "", "cuda"], [12, 2, 1, "", "forward"], [12, 2, 1, "", "from_pretrained"], [12, 2, 1, "", "mps"], [12, 2, 1, "", "run_with_cache"], [12, 2, 1, "", "to"]], "transformer_lens.HookedEncoderDecoder": [[13, 1, 1, "", "HookedEncoderDecoder"]], "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder": [[13, 3, 1, "", "OV"], [13, 3, 1, "", "QK"], [13, 3, 1, "", "W_E"], [13, 3, 1, "", "W_K"], [13, 3, 1, "", "W_O"], [13, 3, 1, "", "W_Q"], [13, 3, 1, "", "W_U"], [13, 3, 1, "", "W_V"], [13, 3, 1, "", "W_in"], [13, 3, 1, "", "W_out"], [13, 3, 1, "", "W_pos"], [13, 2, 1, "", "all_head_labels"], [13, 3, 1, "", "b_K"], [13, 3, 1, "", "b_O"], [13, 3, 1, "", "b_Q"], [13, 3, 1, "", "b_U"], [13, 3, 1, "", "b_V"], [13, 3, 1, "", "b_in"], [13, 3, 1, "", "b_out"], [13, 2, 1, "", "cpu"], [13, 2, 1, "", "cuda"], [13, 2, 1, "", "forward"], [13, 2, 1, "", "from_pretrained"], [13, 2, 1, "", "mps"], [13, 2, 1, "", "run_with_cache"], [13, 2, 1, "", "to"]], "transformer_lens.HookedTransformer": [[14, 1, 1, "", "HookedTransformer"], [14, 1, 1, "", "Output"]], "transformer_lens.HookedTransformer.HookedTransformer": [[14, 3, 1, "", "OV"], [14, 3, 1, "", "QK"], [14, 3, 1, "", "W_E"], [14, 3, 1, "", "W_E_pos"], [14, 3, 1, "", "W_K"], [14, 3, 1, "", "W_O"], [14, 3, 1, "", "W_Q"], [14, 3, 1, "", "W_U"], [14, 3, 1, "", "W_V"], [14, 3, 1, "", "W_gate"], [14, 3, 1, "", "W_in"], [14, 3, 1, "", "W_out"], [14, 3, 1, "", "W_pos"], [14, 2, 1, "", "__init__"], [14, 2, 1, "", "accumulated_bias"], [14, 2, 1, "", "all_composition_scores"], [14, 2, 1, "", "all_head_labels"], [14, 3, 1, "", "b_K"], [14, 3, 1, "", "b_O"], [14, 3, 1, "", "b_Q"], [14, 3, 1, "", "b_U"], [14, 3, 1, "", "b_V"], [14, 3, 1, "", "b_in"], [14, 3, 1, "", "b_out"], [14, 2, 1, "", "center_unembed"], [14, 2, 1, "", "center_writing_weights"], [14, 2, 1, "", "check_hooks_to_add"], [14, 2, 1, "", "cpu"], [14, 2, 1, "", "cuda"], [14, 2, 1, "", "fold_layer_norm"], [14, 2, 1, "", "fold_value_biases"], [14, 2, 1, "", "forward"], [14, 2, 1, "", "from_pretrained"], [14, 2, 1, "", "from_pretrained_no_processing"], [14, 2, 1, "", "generate"], [14, 2, 1, "", "get_token_position"], [14, 2, 1, "", "init_weights"], [14, 2, 1, "", "input_to_embed"], [14, 4, 1, "", "ln_final"], [14, 2, 1, "", "load_and_process_state_dict"], [14, 2, 1, "", "load_sample_training_dataset"], [14, 2, 1, "", "loss_fn"], [14, 2, 1, "", "move_model_modules_to_device"], [14, 2, 1, "", "mps"], [14, 2, 1, "", "process_weights_"], [14, 2, 1, "", "refactor_factored_attn_matrices"], [14, 2, 1, "", "run_with_cache"], [14, 2, 1, "", "sample_datapoint"], [14, 2, 1, "", "set_tokenizer"], [14, 2, 1, "", "set_use_attn_in"], [14, 2, 1, "", "set_use_attn_result"], [14, 2, 1, "", "set_use_hook_mlp_in"], [14, 2, 1, "", "set_use_split_qkv_input"], [14, 2, 1, "", "to"], [14, 2, 1, "", "to_single_str_token"], [14, 2, 1, "", "to_single_token"], [14, 2, 1, "", "to_str_tokens"], [14, 2, 1, "", "to_string"], [14, 2, 1, "", "to_tokens"], [14, 2, 1, "", "tokens_to_residual_directions"]], "transformer_lens.HookedTransformer.Output": [[14, 4, 1, "", "logits"], [14, 4, 1, "", "loss"]], "transformer_lens.HookedTransformerConfig": [[15, 1, 1, "", "HookedTransformerConfig"]], "transformer_lens.HookedTransformerConfig.HookedTransformerConfig": [[15, 4, 1, "", "act_fn"], [15, 4, 1, "", "attention_dir"], [15, 4, 1, "", "attn_only"], [15, 4, 1, "", "attn_scale"], [15, 4, 1, "", "attn_scores_soft_cap"], [15, 4, 1, "", "attn_types"], [15, 4, 1, "", "checkpoint_index"], [15, 4, 1, "", "checkpoint_label_type"], [15, 4, 1, "", "checkpoint_value"], [15, 4, 1, "", "d_head"], [15, 4, 1, "", "d_mlp"], [15, 4, 1, "", "d_model"], [15, 4, 1, "", "d_vocab"], [15, 4, 1, "", "d_vocab_out"], [15, 4, 1, "", "decoder_start_token_id"], [15, 4, 1, "", "default_prepend_bos"], [15, 4, 1, "", "device"], [15, 4, 1, "", "dtype"], [15, 4, 1, "", "eps"], [15, 4, 1, "", "experts_per_token"], [15, 4, 1, "", "final_rms"], [15, 4, 1, "", "from_checkpoint"], [15, 2, 1, "", "from_dict"], [15, 4, 1, "", "gated_mlp"], [15, 4, 1, "", "init_mode"], [15, 4, 1, "", "init_weights"], [15, 4, 1, "", "initializer_range"], [15, 2, 1, "", "is_layer_norm_activation"], [15, 4, 1, "", "load_in_4bit"], [15, 4, 1, "", "model_name"], [15, 4, 1, "", "n_ctx"], [15, 4, 1, "", "n_devices"], [15, 4, 1, "", "n_heads"], [15, 4, 1, "", "n_key_value_heads"], [15, 4, 1, "", "n_layers"], [15, 4, 1, "", "n_params"], [15, 4, 1, "", "normalization_type"], [15, 4, 1, "", "num_experts"], [15, 4, 1, "", "original_architecture"], [15, 4, 1, "", "output_logits_soft_cap"], [15, 4, 1, "", "parallel_attn_mlp"], [15, 4, 1, "", "positional_embedding_type"], [15, 4, 1, "", "post_embedding_ln"], [15, 4, 1, "", "relative_attention_max_distance"], [15, 4, 1, "", "relative_attention_num_buckets"], [15, 4, 1, "", "rotary_adjacent_pairs"], [15, 4, 1, "", "rotary_base"], [15, 4, 1, "", "rotary_dim"], [15, 4, 1, "", "scale_attn_by_inverse_layer_idx"], [15, 4, 1, "", "seed"], [15, 2, 1, "", "set_seed_everywhere"], [15, 4, 1, "", "tie_word_embeddings"], [15, 2, 1, "", "to_dict"], [15, 4, 1, "", "tokenizer_name"], [15, 4, 1, "", "tokenizer_prepends_bos"], [15, 4, 1, "", "trust_remote_code"], [15, 2, 1, "", "unwrap"], [15, 4, 1, "", "use_attn_in"], [15, 4, 1, "", "use_attn_result"], [15, 4, 1, "", "use_attn_scale"], [15, 4, 1, "", "use_hook_mlp_in"], [15, 4, 1, "", "use_hook_tokens"], [15, 4, 1, "", "use_local_attn"], [15, 4, 1, "", "use_normalization_before_and_after"], [15, 4, 1, "", "use_split_qkv_input"], [15, 4, 1, "", "window_size"]], "transformer_lens.SVDInterpreter": [[16, 1, 1, "", "SVDInterpreter"]], "transformer_lens.SVDInterpreter.SVDInterpreter": [[16, 2, 1, "", "get_singular_vectors"]], "transformer_lens.components": [[18, 0, 0, "-", "abstract_attention"], [19, 0, 0, "-", "attention"], [20, 0, 0, "-", "bert_block"], [21, 0, 0, "-", "bert_embed"], [22, 0, 0, "-", "bert_mlm_head"], [23, 0, 0, "-", "embed"], [24, 0, 0, "-", "grouped_query_attention"], [25, 0, 0, "-", "layer_norm"], [26, 0, 0, "-", "layer_norm_pre"], [27, 0, 0, "-", "pos_embed"], [28, 0, 0, "-", "rms_norm"], [29, 0, 0, "-", "rms_norm_pre"], [30, 0, 0, "-", "t5_attention"], [31, 0, 0, "-", "t5_block"], [32, 0, 0, "-", "token_typed_embed"], [33, 0, 0, "-", "transformer_block"], [34, 0, 0, "-", "unembed"]], "transformer_lens.components.abstract_attention": [[18, 1, 1, "", "AbstractAttention"]], "transformer_lens.components.abstract_attention.AbstractAttention": [[18, 3, 1, "", "OV"], [18, 3, 1, "", "QK"], [18, 2, 1, "", "__init__"], [18, 4, 1, "", "alibi"], [18, 2, 1, "", "apply_causal_mask"], [18, 2, 1, "", "apply_rotary"], [18, 2, 1, "", "calculate_attention_scores"], [18, 2, 1, "", "calculate_qkv_matrices"], [18, 2, 1, "", "calculate_sin_cos_rotary"], [18, 2, 1, "", "calculate_z_scores"], [18, 2, 1, "", "create_alibi_bias"], [18, 2, 1, "", "create_alibi_multipliers"], [18, 2, 1, "", "create_alibi_slope"], [18, 2, 1, "", "forward"], [18, 2, 1, "", "rotate_every_two"]], "transformer_lens.components.attention": [[19, 1, 1, "", "Attention"]], "transformer_lens.components.attention.Attention": [[19, 2, 1, "", "__init__"]], "transformer_lens.components.bert_block": [[20, 1, 1, "", "BertBlock"]], "transformer_lens.components.bert_block.BertBlock": [[20, 2, 1, "", "forward"]], "transformer_lens.components.bert_embed": [[21, 1, 1, "", "BertEmbed"]], "transformer_lens.components.bert_embed.BertEmbed": [[21, 2, 1, "", "forward"]], "transformer_lens.components.bert_mlm_head": [[22, 1, 1, "", "BertMLMHead"]], "transformer_lens.components.bert_mlm_head.BertMLMHead": [[22, 2, 1, "", "forward"]], "transformer_lens.components.embed": [[23, 1, 1, "", "Embed"]], "transformer_lens.components.embed.Embed": [[23, 2, 1, "", "forward"]], "transformer_lens.components.grouped_query_attention": [[24, 1, 1, "", "GroupedQueryAttention"]], "transformer_lens.components.grouped_query_attention.GroupedQueryAttention": [[24, 3, 1, "", "W_K"], [24, 3, 1, "", "W_V"], [24, 2, 1, "", "__init__"], [24, 3, 1, "", "b_K"], [24, 3, 1, "", "b_V"], [24, 2, 1, "", "calculate_attention_scores"], [24, 2, 1, "", "calculate_qkv_matrices"], [24, 2, 1, "", "calculate_z_scores"]], "transformer_lens.components.layer_norm": [[25, 1, 1, "", "LayerNorm"]], "transformer_lens.components.layer_norm.LayerNorm": [[25, 2, 1, "", "__init__"], [25, 2, 1, "", "forward"]], "transformer_lens.components.layer_norm_pre": [[26, 1, 1, "", "LayerNormPre"]], "transformer_lens.components.layer_norm_pre.LayerNormPre": [[26, 2, 1, "", "__init__"], [26, 2, 1, "", "forward"]], "transformer_lens.components.pos_embed": [[27, 1, 1, "", "PosEmbed"]], "transformer_lens.components.pos_embed.PosEmbed": [[27, 2, 1, "", "forward"]], "transformer_lens.components.rms_norm": [[28, 1, 1, "", "RMSNorm"]], "transformer_lens.components.rms_norm.RMSNorm": [[28, 2, 1, "", "__init__"], [28, 2, 1, "", "forward"]], "transformer_lens.components.rms_norm_pre": [[29, 1, 1, "", "RMSNormPre"]], "transformer_lens.components.rms_norm_pre.RMSNormPre": [[29, 2, 1, "", "__init__"], [29, 2, 1, "", "forward"]], "transformer_lens.components.t5_attention": [[30, 1, 1, "", "T5Attention"]], "transformer_lens.components.t5_attention.T5Attention": [[30, 2, 1, "", "compute_relative_attention_bias"]], "transformer_lens.components.t5_block": [[31, 1, 1, "", "T5Block"]], "transformer_lens.components.t5_block.T5Block": [[31, 2, 1, "", "forward"]], "transformer_lens.components.token_typed_embed": [[32, 1, 1, "", "TokenTypeEmbed"]], "transformer_lens.components.token_typed_embed.TokenTypeEmbed": [[32, 2, 1, "", "forward"]], "transformer_lens.components.transformer_block": [[33, 1, 1, "", "TransformerBlock"]], "transformer_lens.components.transformer_block.TransformerBlock": [[33, 2, 1, "", "apply_mlp"], [33, 2, 1, "", "forward"], [33, 4, 1, "", "ln1"], [33, 4, 1, "", "ln2"], [33, 4, 1, "", "mlp"]], "transformer_lens.components.unembed": [[34, 1, 1, "", "Unembed"]], "transformer_lens.components.unembed.Unembed": [[34, 2, 1, "", "forward"]], "transformer_lens.evals": [[35, 1, 1, "", "IOIDataset"], [35, 5, 1, "", "evaluate"], [35, 5, 1, "", "evaluate_on_dataset"], [35, 5, 1, "", "induction_loss"], [35, 5, 1, "", "ioi_eval"], [35, 5, 1, "", "make_code_data_loader"], [35, 5, 1, "", "make_owt_data_loader"], [35, 5, 1, "", "make_pile_data_loader"], [35, 5, 1, "", "make_wiki_data_loader"], [35, 5, 1, "", "sanity_check"]], "transformer_lens.evals.IOIDataset": [[35, 2, 1, "", "get_default_names"], [35, 2, 1, "", "get_default_nouns"], [35, 2, 1, "", "get_default_templates"], [35, 2, 1, "", "get_sample"]], "transformer_lens.head_detector": [[36, 5, 1, "", "compute_head_attention_similarity_score"], [36, 5, 1, "", "detect_head"], [36, 5, 1, "", "get_duplicate_token_head_detection_pattern"], [36, 5, 1, "", "get_induction_head_detection_pattern"], [36, 5, 1, "", "get_previous_token_head_detection_pattern"], [36, 5, 1, "", "get_supported_heads"]], "transformer_lens.hook_points": [[37, 4, 1, "", "HookFunction"], [37, 1, 1, "", "HookPoint"], [37, 1, 1, "", "HookedRootModule"], [37, 1, 1, "", "LensHandle"]], "transformer_lens.hook_points.HookPoint": [[37, 2, 1, "", "add_hook"], [37, 2, 1, "", "add_perma_hook"], [37, 2, 1, "", "clear_context"], [37, 2, 1, "", "forward"], [37, 2, 1, "", "layer"], [37, 2, 1, "", "remove_hooks"]], "transformer_lens.hook_points.HookedRootModule": [[37, 2, 1, "", "add_caching_hooks"], [37, 2, 1, "", "add_hook"], [37, 2, 1, "", "add_perma_hook"], [37, 2, 1, "", "cache_all"], [37, 2, 1, "", "cache_some"], [37, 2, 1, "", "check_and_add_hook"], [37, 2, 1, "", "check_hooks_to_add"], [37, 2, 1, "", "clear_contexts"], [37, 2, 1, "", "get_caching_hooks"], [37, 4, 1, "", "hook_dict"], [37, 2, 1, "", "hook_points"], [37, 2, 1, "", "hooks"], [37, 4, 1, "", "mod_dict"], [37, 4, 1, "", "name"], [37, 2, 1, "", "remove_all_hook_fns"], [37, 2, 1, "", "reset_hooks"], [37, 2, 1, "", "run_with_cache"], [37, 2, 1, "", "run_with_hooks"], [37, 2, 1, "", "setup"]], "transformer_lens.hook_points.LensHandle": [[37, 4, 1, "", "context_level"], [37, 4, 1, "", "hook"], [37, 4, 1, "", "is_permanent"]], "transformer_lens.loading_from_pretrained": [[38, 1, 1, "", "Config"], [38, 6, 1, "", "MODEL_ALIASES"], [38, 6, 1, "", "NON_HF_HOSTED_MODEL_NAMES"], [38, 6, 1, "", "OFFICIAL_MODEL_NAMES"], [38, 5, 1, "", "get_checkpoint_labels"], [38, 5, 1, "", "get_num_params_of_pretrained"], [38, 5, 1, "", "get_pretrained_model_config"]], "transformer_lens.loading_from_pretrained.Config": [[38, 4, 1, "", "d_head"], [38, 4, 1, "", "d_mlp"], [38, 4, 1, "", "d_model"], [38, 4, 1, "", "d_vocab"], [38, 4, 1, "", "debug"], [38, 4, 1, "", "init_range"], [38, 4, 1, "", "layer_norm_eps"], [38, 4, 1, "", "n_ctx"], [38, 4, 1, "", "n_heads"], [38, 4, 1, "", "n_layers"]], "transformer_lens.past_key_value_caching": [[39, 1, 1, "", "HookedTransformerKeyValueCache"], [39, 1, 1, "", "HookedTransformerKeyValueCacheEntry"]], "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCache": [[39, 2, 1, "", "append_attention_mask"], [39, 4, 1, "", "entries"], [39, 2, 1, "", "freeze"], [39, 4, 1, "", "frozen"], [39, 2, 1, "", "init_cache"], [39, 4, 1, "", "previous_attention_mask"], [39, 2, 1, "", "unfreeze"]], "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCacheEntry": [[39, 2, 1, "", "append"], [39, 4, 1, "", "frozen"], [39, 2, 1, "", "init_cache_entry"], [39, 4, 1, "", "past_keys"], [39, 4, 1, "", "past_values"]], "transformer_lens.patching": [[40, 5, 1, "", "generic_activation_patch"], [40, 5, 1, "", "get_act_patch_attn_head_all_pos_every"], [40, 5, 1, "", "get_act_patch_attn_head_by_pos_every"], [40, 5, 1, "", "get_act_patch_attn_head_k_all_pos"], [40, 5, 1, "", "get_act_patch_attn_head_k_by_pos"], [40, 5, 1, "", "get_act_patch_attn_head_out_all_pos"], [40, 5, 1, "", "get_act_patch_attn_head_out_by_pos"], [40, 5, 1, "", "get_act_patch_attn_head_pattern_all_pos"], [40, 5, 1, "", "get_act_patch_attn_head_pattern_by_pos"], [40, 5, 1, "", "get_act_patch_attn_head_pattern_dest_src_pos"], [40, 5, 1, "", "get_act_patch_attn_head_q_all_pos"], [40, 5, 1, "", "get_act_patch_attn_head_q_by_pos"], [40, 5, 1, "", "get_act_patch_attn_head_v_all_pos"], [40, 5, 1, "", "get_act_patch_attn_head_v_by_pos"], [40, 5, 1, "", "get_act_patch_attn_out"], [40, 5, 1, "", "get_act_patch_block_every"], [40, 5, 1, "", "get_act_patch_mlp_out"], [40, 5, 1, "", "get_act_patch_resid_mid"], [40, 5, 1, "", "get_act_patch_resid_pre"], [40, 5, 1, "", "layer_head_dest_src_pos_pattern_patch_setter"], [40, 5, 1, "", "layer_head_pattern_patch_setter"], [40, 5, 1, "", "layer_head_pos_pattern_patch_setter"], [40, 5, 1, "", "layer_head_vector_patch_setter"], [40, 5, 1, "", "layer_pos_head_vector_patch_setter"], [40, 5, 1, "", "layer_pos_patch_setter"]], "transformer_lens.pretrained.weight_conversions": [[43, 0, 0, "-", "bert"], [44, 0, 0, "-", "bloom"], [45, 0, 0, "-", "coder"], [46, 0, 0, "-", "gemma"], [47, 0, 0, "-", "gpt2"], [48, 0, 0, "-", "gptj"], [49, 0, 0, "-", "llama"], [50, 0, 0, "-", "mingpt"], [51, 0, 0, "-", "mistral"], [52, 0, 0, "-", "mixtral"], [53, 0, 0, "-", "nanogpt"], [54, 0, 0, "-", "neel_solu_old"], [55, 0, 0, "-", "neo"], [56, 0, 0, "-", "neox"], [57, 0, 0, "-", "opt"], [58, 0, 0, "-", "phi"], [59, 0, 0, "-", "phi3"], [60, 0, 0, "-", "qwen"], [61, 0, 0, "-", "qwen2"], [62, 0, 0, "-", "t5"]], "transformer_lens.pretrained.weight_conversions.bloom": [[44, 5, 1, "", "convert_bloom_weights"]], "transformer_lens.pretrained.weight_conversions.coder": [[45, 5, 1, "", "convert_coder_weights"]], "transformer_lens.pretrained.weight_conversions.mistral": [[51, 5, 1, "", "convert_mistral_weights"]], "transformer_lens.pretrained.weight_conversions.mixtral": [[52, 5, 1, "", "convert_mixtral_weights"]], "transformer_lens.pretrained.weight_conversions.phi": [[58, 5, 1, "", "convert_phi_weights"]], "transformer_lens.pretrained.weight_conversions.phi3": [[59, 5, 1, "", "convert_phi3_weights"]], "transformer_lens.pretrained.weight_conversions.qwen": [[60, 5, 1, "", "convert_qwen_weights"]], "transformer_lens.pretrained.weight_conversions.qwen2": [[61, 5, 1, "", "convert_qwen2_weights"]], "transformer_lens.pretrained.weight_conversions.t5": [[62, 5, 1, "", "convert_t5_weights"]], "transformer_lens.train": [[63, 1, 1, "", "HookedTransformerTrainConfig"], [63, 5, 1, "", "train"]], "transformer_lens.train.HookedTransformerTrainConfig": [[63, 4, 1, "", "batch_size"], [63, 4, 1, "", "device"], [63, 4, 1, "", "lr"], [63, 4, 1, "", "max_grad_norm"], [63, 4, 1, "", "max_steps"], [63, 4, 1, "", "momentum"], [63, 4, 1, "", "num_epochs"], [63, 4, 1, "", "optimizer_name"], [63, 4, 1, "", "print_every"], [63, 4, 1, "", "save_dir"], [63, 4, 1, "", "save_every"], [63, 4, 1, "", "seed"], [63, 4, 1, "", "wandb"], [63, 4, 1, "", "wandb_project_name"], [63, 4, 1, "", "warmup_steps"], [63, 4, 1, "", "weight_decay"]], "transformer_lens.utilities": [[65, 0, 0, "-", "activation_functions"], [66, 0, 0, "-", "addmm"], [67, 0, 0, "-", "attention"], [68, 0, 0, "-", "devices"]], "transformer_lens.utilities.addmm": [[66, 5, 1, "", "batch_addmm"], [66, 5, 1, "", "vanilla_addmm"]], "transformer_lens.utilities.attention": [[67, 5, 1, "", "complex_attn_linear"], [67, 5, 1, "", "simple_attn_linear"]], "transformer_lens.utilities.devices": [[68, 5, 1, "", "get_device_for_block_index"], [68, 5, 1, "", "move_to_and_update_config"]], "transformer_lens.utils": [[69, 1, 1, "", "LocallyOverridenDefaults"], [69, 1, 1, "", "Slice"], [69, 6, 1, "", "SliceInput"], [69, 5, 1, "", "calc_fan_in_and_fan_out"], [69, 5, 1, "", "composition_scores"], [69, 5, 1, "", "download_file_from_hf"], [69, 5, 1, "", "gelu_fast"], [69, 5, 1, "", "gelu_new"], [69, 5, 1, "", "get_act_name"], [69, 5, 1, "", "get_attention_mask"], [69, 5, 1, "", "get_corner"], [69, 5, 1, "", "get_cumsum_along_dim"], [69, 5, 1, "", "get_dataset"], [69, 5, 1, "", "get_device"], [69, 5, 1, "", "get_input_with_manually_prepended_bos"], [69, 5, 1, "", "get_nested_attr"], [69, 5, 1, "", "get_offset_position_ids"], [69, 5, 1, "", "get_tokenizer_with_bos"], [69, 5, 1, "", "get_tokens_with_bos_removed"], [69, 5, 1, "", "init_kaiming_normal_"], [69, 5, 1, "", "init_kaiming_uniform_"], [69, 5, 1, "", "init_xavier_normal_"], [69, 5, 1, "", "init_xavier_uniform_"], [69, 5, 1, "", "is_lower_triangular"], [69, 5, 1, "", "is_square"], [69, 5, 1, "", "keep_single_column"], [69, 5, 1, "", "lm_accuracy"], [69, 5, 1, "", "lm_cross_entropy_loss"], [69, 5, 1, "", "override_or_use_default_value"], [69, 5, 1, "", "print_gpu_mem"], [69, 5, 1, "", "remove_batch_dim"], [69, 5, 1, "", "repeat_along_head_dimension"], [69, 5, 1, "", "sample_logits"], [69, 5, 1, "", "set_nested_attr"], [69, 5, 1, "", "solu"], [69, 5, 1, "", "test_prompt"], [69, 5, 1, "", "to_numpy"], [69, 5, 1, "", "tokenize_and_concatenate"], [69, 5, 1, "", "transpose"]], "transformer_lens.utils.LocallyOverridenDefaults": [[69, 2, 1, "", "__init__"]], "transformer_lens.utils.Slice": [[69, 2, 1, "", "__init__"], [69, 2, 1, "", "apply"], [69, 2, 1, "", "indices"], [69, 4, 1, "", "slice"], [69, 2, 1, "", "unwrap"]]}, "objtypes": {"0": "py:module", "1": "py:class", "2": "py:method", "3": "py:property", "4": "py:attribute", "5": "py:function", "6": "py:data"}, "objnames": {"0": ["py", "module", "Python module"], "1": ["py", "class", "Python class"], "2": ["py", "method", "Python method"], "3": ["py", "property", "Python property"], "4": ["py", "attribute", "Python attribute"], "5": ["py", "function", "Python function"], "6": ["py", "data", "Python data"]}, "titleterms": {"citat": 0, "contribut": 1, "setup": [1, 70, 71], "devcontain": 1, "manual": 1, "test": [1, 5], "run": [1, 71], "format": 1, "document": 1, "docstr": 1, "style": 1, "guid": 1, "section": 1, "order": 1, "support": 1, "sphinx": 1, "properti": [1, 72], "refer": 1, "other": [1, 71], "function": [1, 70], "class": [1, 71], "math": 1, "markup": 1, "galleri": 2, "get": [3, 4], "start": [3, 4, 7], "advic": 3, "read": [3, 70], "code": 3, "instal": 3, "huggingfac": 3, "gate": 3, "access": [3, 71], "mechanist": [4, 73], "interpret": [4, 71, 73], "transformerlen": [5, 73], "2": 5, "0": 5, "first": 5, "an": [5, 71], "introduct": [5, 70, 71], "adopt": 5, "semant": 5, "version": 5, "deprec": 5, "roadmap": 5, "immedi": 5, "within": 5, "next": 5, "month": 5, "mid": 5, "term": 5, "3": 5, "perform": 5, "streamlin": 5, "ad": 5, "new": 5, "model": [5, 71, 72, 73], "long": 5, "year": 5, "integr": 5, "contributor": 5, "dev": 5, "branch": 5, "coverag": 5, "compon": [5, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34], "refactor": 5, "conclus": 5, "appendix": 5, "special": 6, "case": 6, "mixtur": 6, "expert": 6, "error": 6, "rate": 6, "tutori": 7, "where": 7, "To": 7, "demo": [7, 70, 71], "transform": [8, 71], "len": [8, 70, 71], "api": 8, "content": 8, "transformer_len": [9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69], "submodul": [9, 17, 42, 64], "subpackag": [9, 41], "activationcach": 10, "factoredmatrix": 11, "hookedencod": 12, "hookedencoderdecod": 13, "hookedtransform": 14, "hookedtransformerconfig": 15, "svdinterpret": 16, "abstract_attent": 18, "attent": [19, 67, 70], "bert_block": 20, "bert_emb": 21, "bert_mlm_head": 22, "emb": 23, "grouped_query_attent": 24, "layer_norm": 25, "layer_norm_pr": 26, "pos_emb": 27, "rms_norm": 28, "rms_norm_pr": 29, "t5_attent": 30, "t5_block": 31, "token_typed_emb": 32, "transformer_block": 33, "unemb": 34, "eval": 35, "head_detector": 36, "hook_point": 37, "loading_from_pretrain": 38, "past_key_value_cach": 39, "patch": [40, 70, 71], "pretrain": [41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62], "weight_convers": [42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62], "bert": 43, "bloom": 44, "coder": 45, "gemma": 46, "gpt2": 47, "gptj": 48, "llama": 49, "mingpt": 50, "mistral": 51, "mixtral": 52, "nanogpt": 53, "neel_solu_old": 54, "neo": 55, "neox": 56, "opt": 57, "phi": 58, "phi3": 59, "qwen": 60, "qwen2": 61, "t5": 62, "train": [63, 71], "util": [64, 65, 66, 67, 68, 69], "activation_funct": 65, "addmm": 66, "devic": 68, "exploratori": 70, "analysi": 70, "tip": 70, "thi": 70, "environ": 70, "ignor": 70, "import": [70, 71], "pytorch": 70, "plot": 70, "helper": 70, "indirect": [70, 71], "object": [70, 71], "identif": [70, 71], "brainstorm": 70, "what": 70, "": 70, "actual": 70, "go": 70, "On": 70, "option": 70, "direct": 70, "logit": 70, "attribut": 70, "layer": 70, "head": [70, 71], "activ": [70, 71], "residu": 70, "stream": 70, "decompos": 70, "consolid": 70, "understand": 70, "visual": 70, "pattern": 70, "compar": 70, "paper": 70, "bonu": 70, "explor": 70, "anomali": 70, "earli": 70, "ar": 70, "induct": [70, 71], "implic": 70, "backup": 70, "name": [70, 71], "mover": 70, "main": 71, "notebook": 71, "load": 71, "cach": 71, "all": 71, "hook": 71, "interven": 71, "task": 71, "avail": 71, "overview": 71, "open": 71, "sourc": 71, "librari": [71, 73], "some": 71, "friendli": 71, "i": 71, "ve": 71, "includ": 71, "resourc": 71, "architectur": 71, "paramet": 71, "fold": 71, "layernorm": 71, "For": 71, "curiou": 71, "featur": 71, "deal": 71, "token": 71, "gotcha": 71, "prepend_bo": 71, "factor": 71, "matrix": 71, "basic": 71, "exampl": 71, "medium": 71, "eigenvalu": 71, "copi": 71, "score": 71, "gener": [71, 73], "text": 71, "point": 71, "toi": 71, "pre": 71, "checkpoint": 71, "phase": 71, "transit": 71, "tabl": 72, "A": 73, "languag": 73}, "envversion": {"sphinx.domains.c": 2, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 8, "sphinx.domains.index": 1, "sphinx.domains.javascript": 2, "sphinx.domains.math": 2, "sphinx.domains.python": 3, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "nbsphinx": 4, "sphinx": 57}, "alltitles": {"Citation": [[0, "citation"]], "Contributing": [[1, "contributing"]], "Setup": [[1, "setup"], [70, "Setup"], [71, "Setup"]], "DevContainer": [[1, "devcontainer"]], "Manual Setup": [[1, "manual-setup"]], "Testing": [[1, "testing"]], "Running the tests": [[1, "running-the-tests"]], "Formatting": [[1, "formatting"]], "Documentation": [[1, "documentation"]], "Docstring Style Guide": [[1, "docstring-style-guide"]], "Sections and Order": [[1, "sections-and-order"]], "Supported Sphinx Properties": [[1, "supported-sphinx-properties"]], "References to Other Functions/Classes": [[1, "references-to-other-functions-classes"]], "Maths": [[1, "maths"]], "Markup": [[1, "markup"]], "Gallery": [[2, "gallery"]], "Getting Started": [[3, "getting-started"]], "Advice for Reading the Code": [[3, "advice-for-reading-the-code"]], "Installation": [[3, "installation"]], "Huggingface Gated Access": [[3, "huggingface-gated-access"]], "Getting Started in Mechanistic Interpretability": [[4, "getting-started-in-mechanistic-interpretability"]], "TransformerLens 2.0": [[5, "transformerlens-2-0"]], "First, an introduction": [[5, "first-an-introduction"]], "Adopting Semantic Versioning": [[5, "adopting-semantic-versioning"]], "Deprecations": [[5, "deprecations"]], "Roadmap": [[5, "roadmap"]], "Immediate - within the next month": [[5, "immediate-within-the-next-month"]], "Mid-term - within the next 3 months": [[5, "mid-term-within-the-next-3-months"]], "Performance": [[5, "performance"]], "Streamlining Adding New Models": [[5, "streamlining-adding-new-models"]], "Long-term - within the next year": [[5, "long-term-within-the-next-year"]], "Model Testing": [[5, "model-testing"]], "Model Integration": [[5, "model-integration"]], "Contributors": [[5, "contributors"]], "New Dev Branches": [[5, "new-dev-branches"]], "Integration Tests": [[5, "integration-tests"]], "Test Coverage": [[5, "test-coverage"]], "Components Refactor": [[5, "components-refactor"]], "Conclusion": [[5, "conclusion"]], "Appendix": [[5, "appendix"]], "Semantic Versioning": [[5, "semantic-versioning"]], "Special Cases": [[6, "special-cases"]], "Mixture of Experts error rates": [[6, "mixture-of-experts-error-rates"]], "Tutorials": [[7, "tutorials"]], "Where To Start": [[7, "where-to-start"]], "Demos": [[7, "demos"]], "Transformer Lens API": [[8, "transformer-lens-api"]], "Contents": [[8, "contents"]], "transformer_lens": [[9, "transformer-lens"]], "Submodules": [[9, "submodules"], [17, "submodules"], [42, "submodules"], [64, "submodules"]], "Subpackages": [[9, "subpackages"], [41, "subpackages"]], "transformer_lens.ActivationCache": [[10, "module-transformer_lens.ActivationCache"]], "transformer_lens.FactoredMatrix": [[11, "module-transformer_lens.FactoredMatrix"]], "transformer_lens.HookedEncoder": [[12, "module-transformer_lens.HookedEncoder"]], "transformer_lens.HookedEncoderDecoder": [[13, "module-transformer_lens.HookedEncoderDecoder"]], "transformer_lens.HookedTransformer": [[14, "module-transformer_lens.HookedTransformer"]], "transformer_lens.HookedTransformerConfig": [[15, "module-transformer_lens.HookedTransformerConfig"]], "transformer_lens.SVDInterpreter": [[16, "module-transformer_lens.SVDInterpreter"]], "transformer_lens.components": [[17, "transformer-lens-components"]], "transformer_lens.components.abstract_attention": [[18, "module-transformer_lens.components.abstract_attention"]], "transformer_lens.components.attention": [[19, "module-transformer_lens.components.attention"]], "transformer_lens.components.bert_block": [[20, "module-transformer_lens.components.bert_block"]], "transformer_lens.components.bert_embed": [[21, "module-transformer_lens.components.bert_embed"]], "transformer_lens.components.bert_mlm_head": [[22, "module-transformer_lens.components.bert_mlm_head"]], "transformer_lens.components.embed": [[23, "module-transformer_lens.components.embed"]], "transformer_lens.components.grouped_query_attention": [[24, "module-transformer_lens.components.grouped_query_attention"]], "transformer_lens.components.layer_norm": [[25, "module-transformer_lens.components.layer_norm"]], "transformer_lens.components.layer_norm_pre": [[26, "module-transformer_lens.components.layer_norm_pre"]], "transformer_lens.components.pos_embed": [[27, "module-transformer_lens.components.pos_embed"]], "transformer_lens.components.rms_norm": [[28, "module-transformer_lens.components.rms_norm"]], "transformer_lens.components.rms_norm_pre": [[29, "module-transformer_lens.components.rms_norm_pre"]], "transformer_lens.components.t5_attention": [[30, "module-transformer_lens.components.t5_attention"]], "transformer_lens.components.t5_block": [[31, "module-transformer_lens.components.t5_block"]], "transformer_lens.components.token_typed_embed": [[32, "module-transformer_lens.components.token_typed_embed"]], "transformer_lens.components.transformer_block": [[33, "module-transformer_lens.components.transformer_block"]], "transformer_lens.components.unembed": [[34, "module-transformer_lens.components.unembed"]], "transformer_lens.evals": [[35, "module-transformer_lens.evals"]], "transformer_lens.head_detector": [[36, "module-transformer_lens.head_detector"]], "transformer_lens.hook_points": [[37, "module-transformer_lens.hook_points"]], "transformer_lens.loading_from_pretrained": [[38, "module-transformer_lens.loading_from_pretrained"]], "transformer_lens.past_key_value_caching": [[39, "module-transformer_lens.past_key_value_caching"]], "transformer_lens.patching": [[40, "module-transformer_lens.patching"]], "transformer_lens.pretrained": [[41, "transformer-lens-pretrained"]], "transformer_lens.pretrained.weight_conversions": [[42, "transformer-lens-pretrained-weight-conversions"]], "transformer_lens.pretrained.weight_conversions.bert": [[43, "module-transformer_lens.pretrained.weight_conversions.bert"]], "transformer_lens.pretrained.weight_conversions.bloom": [[44, "module-transformer_lens.pretrained.weight_conversions.bloom"]], "transformer_lens.pretrained.weight_conversions.coder": [[45, "module-transformer_lens.pretrained.weight_conversions.coder"]], "transformer_lens.pretrained.weight_conversions.gemma": [[46, "module-transformer_lens.pretrained.weight_conversions.gemma"]], "transformer_lens.pretrained.weight_conversions.gpt2": [[47, "module-transformer_lens.pretrained.weight_conversions.gpt2"]], "transformer_lens.pretrained.weight_conversions.gptj": [[48, "module-transformer_lens.pretrained.weight_conversions.gptj"]], "transformer_lens.pretrained.weight_conversions.llama": [[49, "module-transformer_lens.pretrained.weight_conversions.llama"]], "transformer_lens.pretrained.weight_conversions.mingpt": [[50, "module-transformer_lens.pretrained.weight_conversions.mingpt"]], "transformer_lens.pretrained.weight_conversions.mistral": [[51, "module-transformer_lens.pretrained.weight_conversions.mistral"]], "transformer_lens.pretrained.weight_conversions.mixtral": [[52, "module-transformer_lens.pretrained.weight_conversions.mixtral"]], "transformer_lens.pretrained.weight_conversions.nanogpt": [[53, "module-transformer_lens.pretrained.weight_conversions.nanogpt"]], "transformer_lens.pretrained.weight_conversions.neel_solu_old": [[54, "module-transformer_lens.pretrained.weight_conversions.neel_solu_old"]], "transformer_lens.pretrained.weight_conversions.neo": [[55, "module-transformer_lens.pretrained.weight_conversions.neo"]], "transformer_lens.pretrained.weight_conversions.neox": [[56, "module-transformer_lens.pretrained.weight_conversions.neox"]], "transformer_lens.pretrained.weight_conversions.opt": [[57, "module-transformer_lens.pretrained.weight_conversions.opt"]], "transformer_lens.pretrained.weight_conversions.phi": [[58, "module-transformer_lens.pretrained.weight_conversions.phi"]], "transformer_lens.pretrained.weight_conversions.phi3": [[59, "module-transformer_lens.pretrained.weight_conversions.phi3"]], "transformer_lens.pretrained.weight_conversions.qwen": [[60, "module-transformer_lens.pretrained.weight_conversions.qwen"]], "transformer_lens.pretrained.weight_conversions.qwen2": [[61, "module-transformer_lens.pretrained.weight_conversions.qwen2"]], "transformer_lens.pretrained.weight_conversions.t5": [[62, "module-transformer_lens.pretrained.weight_conversions.t5"]], "transformer_lens.train": [[63, "module-transformer_lens.train"]], "transformer_lens.utilities": [[64, "transformer-lens-utilities"]], "transformer_lens.utilities.activation_functions": [[65, "module-transformer_lens.utilities.activation_functions"]], "transformer_lens.utilities.addmm": [[66, "module-transformer_lens.utilities.addmm"]], "transformer_lens.utilities.attention": [[67, "module-transformer_lens.utilities.attention"]], "transformer_lens.utilities.devices": [[68, "module-transformer_lens.utilities.devices"]], "transformer_lens.utils": [[69, "module-transformer_lens.utils"]], "Exploratory Analysis Demo": [[70, "Exploratory-Analysis-Demo"]], "Tips for Reading This": [[70, "Tips-for-Reading-This"]], "Environment Setup (ignore)": [[70, "Environment-Setup-(ignore)"]], "Imports": [[70, "Imports"]], "PyTorch Setup": [[70, "PyTorch-Setup"]], "Plotting Helper Functions (ignore)": [[70, "Plotting-Helper-Functions-(ignore)"]], "Introduction": [[70, "Introduction"], [71, "Introduction"]], "Indirect Object Identification": [[70, "Indirect-Object-Identification"]], "Brainstorm What\u2019s Actually Going On (Optional)": [[70, "Brainstorm-What's-Actually-Going-On-(Optional)"]], "Direct Logit Attribution": [[70, "Direct-Logit-Attribution"]], "Logit Lens": [[70, "Logit-Lens"]], "Layer Attribution": [[70, "Layer-Attribution"]], "Head Attribution": [[70, "Head-Attribution"]], "Attention Analysis": [[70, "Attention-Analysis"]], "Activation Patching": [[70, "Activation-Patching"]], "Residual Stream": [[70, "Residual-Stream"]], "Layers": [[70, "Layers"]], "Heads": [[70, "Heads"]], "Decomposing Heads": [[70, "Decomposing-Heads"]], "Consolidating Understanding": [[70, "Consolidating-Understanding"]], "Visualizing Attention Patterns": [[70, "Visualizing-Attention-Patterns"]], "Comparing to the Paper": [[70, "Comparing-to-the-Paper"]], "Bonus: Exploring Anomalies": [[70, "Bonus:-Exploring-Anomalies"]], "Early Heads are Induction Heads(?!)": [[70, "Early-Heads-are-Induction-Heads(?!)"]], "Implications": [[70, "Implications"]], "Backup Name Mover Heads": [[70, "Backup-Name-Mover-Heads"]], "Transformer Lens Main Demo Notebook": [[71, "Transformer-Lens-Main-Demo-Notebook"]], "Loading and Running Models": [[71, "Loading-and-Running-Models"]], "Caching all Activations": [[71, "Caching-all-Activations"]], "Hooks: Intervening on Activations": [[71, "Hooks:-Intervening-on-Activations"]], "Activation Patching on the Indirect Object Identification Task": [[71, "Activation-Patching-on-the-Indirect-Object-Identification-Task"]], "Hooks: Accessing Activations": [[71, "Hooks:-Accessing-Activations"]], "Available Models": [[71, "Available-Models"]], "An overview of the important open source models in the library": [[71, "An-overview-of-the-important-open-source-models-in-the-library"]], "An overview of some interpretability-friendly models I\u2019ve trained and included": [[71, "An-overview-of-some-interpretability-friendly-models-I've-trained-and-included"]], "Other Resources:": [[71, "Other-Resources:"]], "Transformer architecture": [[71, "Transformer-architecture"]], "Parameter Names": [[71, "Parameter-Names"]], "Activation + Hook Names": [[71, "Activation-+-Hook-Names"]], "Folding LayerNorm (For the Curious)": [[71, "Folding-LayerNorm-(For-the-Curious)"]], "Features": [[71, "Features"]], "Dealing with tokens": [[71, "Dealing-with-tokens"]], "Gotcha: prepend_bos": [[71, "Gotcha:-prepend_bos"]], "Factored Matrix Class": [[71, "Factored-Matrix-Class"]], "Basic Examples": [[71, "Basic-Examples"]], "Medium Example: Eigenvalue Copying Scores": [[71, "Medium-Example:-Eigenvalue-Copying-Scores"]], "Generating Text": [[71, "Generating-Text"]], "Hook Points": [[71, "Hook-Points"]], "Toy Example": [[71, "Toy-Example"]], "Loading Pre-Trained Checkpoints": [[71, "Loading-Pre-Trained-Checkpoints"]], "Example: Induction Head Phase Transition": [[71, "Example:-Induction-Head-Phase-Transition"]], "Model Properties Table": [[72, "model-properties-table"]], "TransformerLens": [[73, "transformerlens"]], "A Library for Mechanistic Interpretability of Generative Language Models": [[73, "a-library-for-mechanistic-interpretability-of-generative-language-models"]]}, "indexentries": {"activationcache (class in transformer_lens.activationcache)": [[10, "transformer_lens.ActivationCache.ActivationCache"]], "accumulated_resid() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.accumulated_resid"]], "apply_ln_to_stack() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.apply_ln_to_stack"]], "apply_slice_to_batch_dim() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.apply_slice_to_batch_dim"]], "compute_head_results() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.compute_head_results"]], "decompose_resid() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.decompose_resid"]], "get_full_resid_decomposition() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.get_full_resid_decomposition"]], "get_neuron_results() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.get_neuron_results"]], "items() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.items"]], "keys() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.keys"]], "logit_attrs() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.logit_attrs"]], "module": [[10, "module-transformer_lens.ActivationCache"], [11, "module-transformer_lens.FactoredMatrix"], [12, "module-transformer_lens.HookedEncoder"], [13, "module-transformer_lens.HookedEncoderDecoder"], [14, "module-transformer_lens.HookedTransformer"], [15, "module-transformer_lens.HookedTransformerConfig"], [16, "module-transformer_lens.SVDInterpreter"], [18, "module-transformer_lens.components.abstract_attention"], [19, "module-transformer_lens.components.attention"], [20, "module-transformer_lens.components.bert_block"], [21, "module-transformer_lens.components.bert_embed"], [22, "module-transformer_lens.components.bert_mlm_head"], [23, "module-transformer_lens.components.embed"], [24, "module-transformer_lens.components.grouped_query_attention"], [25, "module-transformer_lens.components.layer_norm"], [26, "module-transformer_lens.components.layer_norm_pre"], [27, "module-transformer_lens.components.pos_embed"], [28, "module-transformer_lens.components.rms_norm"], [29, "module-transformer_lens.components.rms_norm_pre"], [30, "module-transformer_lens.components.t5_attention"], [31, "module-transformer_lens.components.t5_block"], [32, "module-transformer_lens.components.token_typed_embed"], [33, "module-transformer_lens.components.transformer_block"], [34, "module-transformer_lens.components.unembed"], [35, "module-transformer_lens.evals"], [36, "module-transformer_lens.head_detector"], [37, "module-transformer_lens.hook_points"], [38, "module-transformer_lens.loading_from_pretrained"], [39, "module-transformer_lens.past_key_value_caching"], [40, "module-transformer_lens.patching"], [43, "module-transformer_lens.pretrained.weight_conversions.bert"], [44, "module-transformer_lens.pretrained.weight_conversions.bloom"], [45, "module-transformer_lens.pretrained.weight_conversions.coder"], [46, "module-transformer_lens.pretrained.weight_conversions.gemma"], [47, "module-transformer_lens.pretrained.weight_conversions.gpt2"], [48, "module-transformer_lens.pretrained.weight_conversions.gptj"], [49, "module-transformer_lens.pretrained.weight_conversions.llama"], [50, "module-transformer_lens.pretrained.weight_conversions.mingpt"], [51, "module-transformer_lens.pretrained.weight_conversions.mistral"], [52, "module-transformer_lens.pretrained.weight_conversions.mixtral"], [53, "module-transformer_lens.pretrained.weight_conversions.nanogpt"], [54, "module-transformer_lens.pretrained.weight_conversions.neel_solu_old"], [55, "module-transformer_lens.pretrained.weight_conversions.neo"], [56, "module-transformer_lens.pretrained.weight_conversions.neox"], [57, "module-transformer_lens.pretrained.weight_conversions.opt"], [58, "module-transformer_lens.pretrained.weight_conversions.phi"], [59, "module-transformer_lens.pretrained.weight_conversions.phi3"], [60, "module-transformer_lens.pretrained.weight_conversions.qwen"], [61, "module-transformer_lens.pretrained.weight_conversions.qwen2"], [62, "module-transformer_lens.pretrained.weight_conversions.t5"], [63, "module-transformer_lens.train"], [65, "module-transformer_lens.utilities.activation_functions"], [66, "module-transformer_lens.utilities.addmm"], [67, "module-transformer_lens.utilities.attention"], [68, "module-transformer_lens.utilities.devices"], [69, "module-transformer_lens.utils"]], "remove_batch_dim() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.remove_batch_dim"]], "stack_activation() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.stack_activation"]], "stack_head_results() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.stack_head_results"]], "stack_neuron_results() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.stack_neuron_results"]], "to() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.to"]], "toggle_autodiff() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.toggle_autodiff"]], "transformer_lens.activationcache": [[10, "module-transformer_lens.ActivationCache"]], "values() (transformer_lens.activationcache.activationcache method)": [[10, "transformer_lens.ActivationCache.ActivationCache.values"]], "ab (transformer_lens.factoredmatrix.factoredmatrix property)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.AB"]], "ba (transformer_lens.factoredmatrix.factoredmatrix property)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.BA"]], "factoredmatrix (class in transformer_lens.factoredmatrix)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix"]], "s (transformer_lens.factoredmatrix.factoredmatrix property)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.S"]], "t (transformer_lens.factoredmatrix.factoredmatrix property)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.T"]], "u (transformer_lens.factoredmatrix.factoredmatrix property)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.U"]], "vh (transformer_lens.factoredmatrix.factoredmatrix property)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.Vh"]], "collapse_l() (transformer_lens.factoredmatrix.factoredmatrix method)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.collapse_l"]], "collapse_r() (transformer_lens.factoredmatrix.factoredmatrix method)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.collapse_r"]], "eigenvalues (transformer_lens.factoredmatrix.factoredmatrix property)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.eigenvalues"]], "get_corner() (transformer_lens.factoredmatrix.factoredmatrix method)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.get_corner"]], "make_even() (transformer_lens.factoredmatrix.factoredmatrix method)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.make_even"]], "ndim (transformer_lens.factoredmatrix.factoredmatrix property)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.ndim"]], "norm() (transformer_lens.factoredmatrix.factoredmatrix method)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.norm"]], "pair (transformer_lens.factoredmatrix.factoredmatrix property)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.pair"]], "svd() (transformer_lens.factoredmatrix.factoredmatrix method)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.svd"]], "transformer_lens.factoredmatrix": [[11, "module-transformer_lens.FactoredMatrix"]], "unsqueeze() (transformer_lens.factoredmatrix.factoredmatrix method)": [[11, "transformer_lens.FactoredMatrix.FactoredMatrix.unsqueeze"]], "hookedencoder (class in transformer_lens.hookedencoder)": [[12, "transformer_lens.HookedEncoder.HookedEncoder"]], "ov (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.OV"]], "qk (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.QK"]], "w_e (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_E"]], "w_e_pos (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_E_pos"]], "w_k (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_K"]], "w_o (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_O"]], "w_q (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_Q"]], "w_u (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_U"]], "w_v (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_V"]], "w_in (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_in"]], "w_out (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_out"]], "w_pos (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.W_pos"]], "all_head_labels() (transformer_lens.hookedencoder.hookedencoder method)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.all_head_labels"]], "b_k (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.b_K"]], "b_o (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.b_O"]], "b_q (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.b_Q"]], "b_u (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.b_U"]], "b_v (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.b_V"]], "b_in (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.b_in"]], "b_out (transformer_lens.hookedencoder.hookedencoder property)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.b_out"]], "cpu() (transformer_lens.hookedencoder.hookedencoder method)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.cpu"]], "cuda() (transformer_lens.hookedencoder.hookedencoder method)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.cuda"]], "forward() (transformer_lens.hookedencoder.hookedencoder method)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.forward"]], "from_pretrained() (transformer_lens.hookedencoder.hookedencoder class method)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.from_pretrained"]], "mps() (transformer_lens.hookedencoder.hookedencoder method)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.mps"]], "run_with_cache() (transformer_lens.hookedencoder.hookedencoder method)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.run_with_cache"]], "to() (transformer_lens.hookedencoder.hookedencoder method)": [[12, "transformer_lens.HookedEncoder.HookedEncoder.to"]], "transformer_lens.hookedencoder": [[12, "module-transformer_lens.HookedEncoder"]], "hookedencoderdecoder (class in transformer_lens.hookedencoderdecoder)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder"]], "ov (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.OV"]], "qk (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.QK"]], "w_e (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.W_E"]], "w_k (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.W_K"]], "w_o (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.W_O"]], "w_q (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.W_Q"]], "w_u (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.W_U"]], "w_v (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.W_V"]], "w_in (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.W_in"]], "w_out (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.W_out"]], "w_pos (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.W_pos"]], "all_head_labels() (transformer_lens.hookedencoderdecoder.hookedencoderdecoder method)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.all_head_labels"]], "b_k (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.b_K"]], "b_o (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.b_O"]], "b_q (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.b_Q"]], "b_u (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.b_U"]], "b_v (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.b_V"]], "b_in (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.b_in"]], "b_out (transformer_lens.hookedencoderdecoder.hookedencoderdecoder property)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.b_out"]], "cpu() (transformer_lens.hookedencoderdecoder.hookedencoderdecoder method)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.cpu"]], "cuda() (transformer_lens.hookedencoderdecoder.hookedencoderdecoder method)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.cuda"]], "forward() (transformer_lens.hookedencoderdecoder.hookedencoderdecoder method)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.forward"]], "from_pretrained() (transformer_lens.hookedencoderdecoder.hookedencoderdecoder class method)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.from_pretrained"]], "mps() (transformer_lens.hookedencoderdecoder.hookedencoderdecoder method)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.mps"]], "run_with_cache() (transformer_lens.hookedencoderdecoder.hookedencoderdecoder method)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.run_with_cache"]], "to() (transformer_lens.hookedencoderdecoder.hookedencoderdecoder method)": [[13, "transformer_lens.HookedEncoderDecoder.HookedEncoderDecoder.to"]], "transformer_lens.hookedencoderdecoder": [[13, "module-transformer_lens.HookedEncoderDecoder"]], "hookedtransformer (class in transformer_lens.hookedtransformer)": [[14, "transformer_lens.HookedTransformer.HookedTransformer"]], "ov (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.OV"]], "output (class in transformer_lens.hookedtransformer)": [[14, "transformer_lens.HookedTransformer.Output"]], "qk (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.QK"]], "w_e (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_E"]], "w_e_pos (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_E_pos"]], "w_k (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_K"]], "w_o (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_O"]], "w_q (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_Q"]], "w_u (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_U"]], "w_v (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_V"]], "w_gate (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_gate"]], "w_in (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_in"]], "w_out (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_out"]], "w_pos (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.W_pos"]], "__init__() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.__init__"]], "accumulated_bias() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.accumulated_bias"]], "all_composition_scores() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.all_composition_scores"]], "all_head_labels() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.all_head_labels"]], "b_k (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.b_K"]], "b_o (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.b_O"]], "b_q (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.b_Q"]], "b_u (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.b_U"]], "b_v (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.b_V"]], "b_in (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.b_in"]], "b_out (transformer_lens.hookedtransformer.hookedtransformer property)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.b_out"]], "center_unembed() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.center_unembed"]], "center_writing_weights() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.center_writing_weights"]], "check_hooks_to_add() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.check_hooks_to_add"]], "cpu() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.cpu"]], "cuda() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.cuda"]], "fold_layer_norm() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.fold_layer_norm"]], "fold_value_biases() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.fold_value_biases"]], "forward() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.forward"]], "from_pretrained() (transformer_lens.hookedtransformer.hookedtransformer class method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.from_pretrained"]], "from_pretrained_no_processing() (transformer_lens.hookedtransformer.hookedtransformer class method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.from_pretrained_no_processing"]], "generate() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.generate"]], "get_token_position() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.get_token_position"]], "init_weights() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.init_weights"]], "input_to_embed() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.input_to_embed"]], "ln_final (transformer_lens.hookedtransformer.hookedtransformer attribute)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.ln_final"]], "load_and_process_state_dict() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.load_and_process_state_dict"]], "load_sample_training_dataset() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.load_sample_training_dataset"]], "logits (transformer_lens.hookedtransformer.output attribute)": [[14, "transformer_lens.HookedTransformer.Output.logits"]], "loss (transformer_lens.hookedtransformer.output attribute)": [[14, "transformer_lens.HookedTransformer.Output.loss"]], "loss_fn() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.loss_fn"]], "move_model_modules_to_device() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.move_model_modules_to_device"]], "mps() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.mps"]], "process_weights_() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.process_weights_"]], "refactor_factored_attn_matrices() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.refactor_factored_attn_matrices"]], "run_with_cache() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.run_with_cache"]], "sample_datapoint() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.sample_datapoint"]], "set_tokenizer() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.set_tokenizer"]], "set_use_attn_in() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.set_use_attn_in"]], "set_use_attn_result() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.set_use_attn_result"]], "set_use_hook_mlp_in() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.set_use_hook_mlp_in"]], "set_use_split_qkv_input() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.set_use_split_qkv_input"]], "to() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.to"]], "to_single_str_token() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.to_single_str_token"]], "to_single_token() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.to_single_token"]], "to_str_tokens() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.to_str_tokens"]], "to_string() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.to_string"]], "to_tokens() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.to_tokens"]], "tokens_to_residual_directions() (transformer_lens.hookedtransformer.hookedtransformer method)": [[14, "transformer_lens.HookedTransformer.HookedTransformer.tokens_to_residual_directions"]], "transformer_lens.hookedtransformer": [[14, "module-transformer_lens.HookedTransformer"]], "hookedtransformerconfig (class in transformer_lens.hookedtransformerconfig)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig"]], "act_fn (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.act_fn"]], "attention_dir (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.attention_dir"]], "attn_only (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.attn_only"]], "attn_scale (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.attn_scale"]], "attn_scores_soft_cap (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.attn_scores_soft_cap"]], "attn_types (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.attn_types"]], "checkpoint_index (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.checkpoint_index"]], "checkpoint_label_type (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.checkpoint_label_type"]], "checkpoint_value (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.checkpoint_value"]], "d_head (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.d_head"]], "d_mlp (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.d_mlp"]], "d_model (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.d_model"]], "d_vocab (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.d_vocab"]], "d_vocab_out (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.d_vocab_out"]], "decoder_start_token_id (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.decoder_start_token_id"]], "default_prepend_bos (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.default_prepend_bos"]], "device (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.device"]], "dtype (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.dtype"]], "eps (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.eps"]], "experts_per_token (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.experts_per_token"]], "final_rms (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.final_rms"]], "from_checkpoint (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.from_checkpoint"]], "from_dict() (transformer_lens.hookedtransformerconfig.hookedtransformerconfig class method)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.from_dict"]], "gated_mlp (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.gated_mlp"]], "init_mode (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.init_mode"]], "init_weights (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.init_weights"]], "initializer_range (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.initializer_range"]], "is_layer_norm_activation() (transformer_lens.hookedtransformerconfig.hookedtransformerconfig method)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.is_layer_norm_activation"]], "load_in_4bit (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.load_in_4bit"]], "model_name (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.model_name"]], "n_ctx (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.n_ctx"]], "n_devices (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.n_devices"]], "n_heads (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.n_heads"]], "n_key_value_heads (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.n_key_value_heads"]], "n_layers (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.n_layers"]], "n_params (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.n_params"]], "normalization_type (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.normalization_type"]], "num_experts (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.num_experts"]], "original_architecture (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.original_architecture"]], "output_logits_soft_cap (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.output_logits_soft_cap"]], "parallel_attn_mlp (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.parallel_attn_mlp"]], "positional_embedding_type (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.positional_embedding_type"]], "post_embedding_ln (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.post_embedding_ln"]], "relative_attention_max_distance (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.relative_attention_max_distance"]], "relative_attention_num_buckets (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.relative_attention_num_buckets"]], "rotary_adjacent_pairs (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.rotary_adjacent_pairs"]], "rotary_base (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.rotary_base"]], "rotary_dim (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.rotary_dim"]], "scale_attn_by_inverse_layer_idx (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.scale_attn_by_inverse_layer_idx"]], "seed (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.seed"]], "set_seed_everywhere() (transformer_lens.hookedtransformerconfig.hookedtransformerconfig method)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.set_seed_everywhere"]], "tie_word_embeddings (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.tie_word_embeddings"]], "to_dict() (transformer_lens.hookedtransformerconfig.hookedtransformerconfig method)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.to_dict"]], "tokenizer_name (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.tokenizer_name"]], "tokenizer_prepends_bos (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.tokenizer_prepends_bos"]], "transformer_lens.hookedtransformerconfig": [[15, "module-transformer_lens.HookedTransformerConfig"]], "trust_remote_code (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.trust_remote_code"]], "unwrap() (transformer_lens.hookedtransformerconfig.hookedtransformerconfig class method)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.unwrap"]], "use_attn_in (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.use_attn_in"]], "use_attn_result (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.use_attn_result"]], "use_attn_scale (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.use_attn_scale"]], "use_hook_mlp_in (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.use_hook_mlp_in"]], "use_hook_tokens (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.use_hook_tokens"]], "use_local_attn (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.use_local_attn"]], "use_normalization_before_and_after (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.use_normalization_before_and_after"]], "use_split_qkv_input (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.use_split_qkv_input"]], "window_size (transformer_lens.hookedtransformerconfig.hookedtransformerconfig attribute)": [[15, "transformer_lens.HookedTransformerConfig.HookedTransformerConfig.window_size"]], "svdinterpreter (class in transformer_lens.svdinterpreter)": [[16, "transformer_lens.SVDInterpreter.SVDInterpreter"]], "get_singular_vectors() (transformer_lens.svdinterpreter.svdinterpreter method)": [[16, "transformer_lens.SVDInterpreter.SVDInterpreter.get_singular_vectors"]], "transformer_lens.svdinterpreter": [[16, "module-transformer_lens.SVDInterpreter"]], "abstractattention (class in transformer_lens.components.abstract_attention)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention"]], "ov (transformer_lens.components.abstract_attention.abstractattention property)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.OV"]], "qk (transformer_lens.components.abstract_attention.abstractattention property)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.QK"]], "__init__() (transformer_lens.components.abstract_attention.abstractattention method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.__init__"]], "alibi (transformer_lens.components.abstract_attention.abstractattention attribute)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.alibi"]], "apply_causal_mask() (transformer_lens.components.abstract_attention.abstractattention method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.apply_causal_mask"]], "apply_rotary() (transformer_lens.components.abstract_attention.abstractattention method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.apply_rotary"]], "calculate_attention_scores() (transformer_lens.components.abstract_attention.abstractattention method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.calculate_attention_scores"]], "calculate_qkv_matrices() (transformer_lens.components.abstract_attention.abstractattention method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.calculate_qkv_matrices"]], "calculate_sin_cos_rotary() (transformer_lens.components.abstract_attention.abstractattention method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.calculate_sin_cos_rotary"]], "calculate_z_scores() (transformer_lens.components.abstract_attention.abstractattention method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.calculate_z_scores"]], "create_alibi_bias() (transformer_lens.components.abstract_attention.abstractattention static method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.create_alibi_bias"]], "create_alibi_multipliers() (transformer_lens.components.abstract_attention.abstractattention static method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.create_alibi_multipliers"]], "create_alibi_slope() (transformer_lens.components.abstract_attention.abstractattention static method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.create_alibi_slope"]], "forward() (transformer_lens.components.abstract_attention.abstractattention method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.forward"]], "rotate_every_two() (transformer_lens.components.abstract_attention.abstractattention method)": [[18, "transformer_lens.components.abstract_attention.AbstractAttention.rotate_every_two"]], "transformer_lens.components.abstract_attention": [[18, "module-transformer_lens.components.abstract_attention"]], "attention (class in transformer_lens.components.attention)": [[19, "transformer_lens.components.attention.Attention"]], "__init__() (transformer_lens.components.attention.attention method)": [[19, "transformer_lens.components.attention.Attention.__init__"]], "transformer_lens.components.attention": [[19, "module-transformer_lens.components.attention"]], "bertblock (class in transformer_lens.components.bert_block)": [[20, "transformer_lens.components.bert_block.BertBlock"]], "forward() (transformer_lens.components.bert_block.bertblock method)": [[20, "transformer_lens.components.bert_block.BertBlock.forward"]], "transformer_lens.components.bert_block": [[20, "module-transformer_lens.components.bert_block"]], "bertembed (class in transformer_lens.components.bert_embed)": [[21, "transformer_lens.components.bert_embed.BertEmbed"]], "forward() (transformer_lens.components.bert_embed.bertembed method)": [[21, "transformer_lens.components.bert_embed.BertEmbed.forward"]], "transformer_lens.components.bert_embed": [[21, "module-transformer_lens.components.bert_embed"]], "bertmlmhead (class in transformer_lens.components.bert_mlm_head)": [[22, "transformer_lens.components.bert_mlm_head.BertMLMHead"]], "forward() (transformer_lens.components.bert_mlm_head.bertmlmhead method)": [[22, "transformer_lens.components.bert_mlm_head.BertMLMHead.forward"]], "transformer_lens.components.bert_mlm_head": [[22, "module-transformer_lens.components.bert_mlm_head"]], "embed (class in transformer_lens.components.embed)": [[23, "transformer_lens.components.embed.Embed"]], "forward() (transformer_lens.components.embed.embed method)": [[23, "transformer_lens.components.embed.Embed.forward"]], "transformer_lens.components.embed": [[23, "module-transformer_lens.components.embed"]], "groupedqueryattention (class in transformer_lens.components.grouped_query_attention)": [[24, "transformer_lens.components.grouped_query_attention.GroupedQueryAttention"]], "w_k (transformer_lens.components.grouped_query_attention.groupedqueryattention property)": [[24, "transformer_lens.components.grouped_query_attention.GroupedQueryAttention.W_K"]], "w_v (transformer_lens.components.grouped_query_attention.groupedqueryattention property)": [[24, "transformer_lens.components.grouped_query_attention.GroupedQueryAttention.W_V"]], "__init__() (transformer_lens.components.grouped_query_attention.groupedqueryattention method)": [[24, "transformer_lens.components.grouped_query_attention.GroupedQueryAttention.__init__"]], "b_k (transformer_lens.components.grouped_query_attention.groupedqueryattention property)": [[24, "transformer_lens.components.grouped_query_attention.GroupedQueryAttention.b_K"]], "b_v (transformer_lens.components.grouped_query_attention.groupedqueryattention property)": [[24, "transformer_lens.components.grouped_query_attention.GroupedQueryAttention.b_V"]], "calculate_attention_scores() (transformer_lens.components.grouped_query_attention.groupedqueryattention method)": [[24, "transformer_lens.components.grouped_query_attention.GroupedQueryAttention.calculate_attention_scores"]], "calculate_qkv_matrices() (transformer_lens.components.grouped_query_attention.groupedqueryattention method)": [[24, "transformer_lens.components.grouped_query_attention.GroupedQueryAttention.calculate_qkv_matrices"]], "calculate_z_scores() (transformer_lens.components.grouped_query_attention.groupedqueryattention method)": [[24, "transformer_lens.components.grouped_query_attention.GroupedQueryAttention.calculate_z_scores"]], "transformer_lens.components.grouped_query_attention": [[24, "module-transformer_lens.components.grouped_query_attention"]], "layernorm (class in transformer_lens.components.layer_norm)": [[25, "transformer_lens.components.layer_norm.LayerNorm"]], "__init__() (transformer_lens.components.layer_norm.layernorm method)": [[25, "transformer_lens.components.layer_norm.LayerNorm.__init__"]], "forward() (transformer_lens.components.layer_norm.layernorm method)": [[25, "transformer_lens.components.layer_norm.LayerNorm.forward"]], "transformer_lens.components.layer_norm": [[25, "module-transformer_lens.components.layer_norm"]], "layernormpre (class in transformer_lens.components.layer_norm_pre)": [[26, "transformer_lens.components.layer_norm_pre.LayerNormPre"]], "__init__() (transformer_lens.components.layer_norm_pre.layernormpre method)": [[26, "transformer_lens.components.layer_norm_pre.LayerNormPre.__init__"]], "forward() (transformer_lens.components.layer_norm_pre.layernormpre method)": [[26, "transformer_lens.components.layer_norm_pre.LayerNormPre.forward"]], "transformer_lens.components.layer_norm_pre": [[26, "module-transformer_lens.components.layer_norm_pre"]], "posembed (class in transformer_lens.components.pos_embed)": [[27, "transformer_lens.components.pos_embed.PosEmbed"]], "forward() (transformer_lens.components.pos_embed.posembed method)": [[27, "transformer_lens.components.pos_embed.PosEmbed.forward"]], "transformer_lens.components.pos_embed": [[27, "module-transformer_lens.components.pos_embed"]], "rmsnorm (class in transformer_lens.components.rms_norm)": [[28, "transformer_lens.components.rms_norm.RMSNorm"]], "__init__() (transformer_lens.components.rms_norm.rmsnorm method)": [[28, "transformer_lens.components.rms_norm.RMSNorm.__init__"]], "forward() (transformer_lens.components.rms_norm.rmsnorm method)": [[28, "transformer_lens.components.rms_norm.RMSNorm.forward"]], "transformer_lens.components.rms_norm": [[28, "module-transformer_lens.components.rms_norm"]], "rmsnormpre (class in transformer_lens.components.rms_norm_pre)": [[29, "transformer_lens.components.rms_norm_pre.RMSNormPre"]], "__init__() (transformer_lens.components.rms_norm_pre.rmsnormpre method)": [[29, "transformer_lens.components.rms_norm_pre.RMSNormPre.__init__"]], "forward() (transformer_lens.components.rms_norm_pre.rmsnormpre method)": [[29, "transformer_lens.components.rms_norm_pre.RMSNormPre.forward"]], "transformer_lens.components.rms_norm_pre": [[29, "module-transformer_lens.components.rms_norm_pre"]], "t5attention (class in transformer_lens.components.t5_attention)": [[30, "transformer_lens.components.t5_attention.T5Attention"]], "compute_relative_attention_bias() (transformer_lens.components.t5_attention.t5attention method)": [[30, "transformer_lens.components.t5_attention.T5Attention.compute_relative_attention_bias"]], "transformer_lens.components.t5_attention": [[30, "module-transformer_lens.components.t5_attention"]], "t5block (class in transformer_lens.components.t5_block)": [[31, "transformer_lens.components.t5_block.T5Block"]], "forward() (transformer_lens.components.t5_block.t5block method)": [[31, "transformer_lens.components.t5_block.T5Block.forward"]], "transformer_lens.components.t5_block": [[31, "module-transformer_lens.components.t5_block"]], "tokentypeembed (class in transformer_lens.components.token_typed_embed)": [[32, "transformer_lens.components.token_typed_embed.TokenTypeEmbed"]], "forward() (transformer_lens.components.token_typed_embed.tokentypeembed method)": [[32, "transformer_lens.components.token_typed_embed.TokenTypeEmbed.forward"]], "transformer_lens.components.token_typed_embed": [[32, "module-transformer_lens.components.token_typed_embed"]], "transformerblock (class in transformer_lens.components.transformer_block)": [[33, "transformer_lens.components.transformer_block.TransformerBlock"]], "apply_mlp() (transformer_lens.components.transformer_block.transformerblock method)": [[33, "transformer_lens.components.transformer_block.TransformerBlock.apply_mlp"]], "forward() (transformer_lens.components.transformer_block.transformerblock method)": [[33, "transformer_lens.components.transformer_block.TransformerBlock.forward"]], "ln1 (transformer_lens.components.transformer_block.transformerblock attribute)": [[33, "transformer_lens.components.transformer_block.TransformerBlock.ln1"]], "ln2 (transformer_lens.components.transformer_block.transformerblock attribute)": [[33, "transformer_lens.components.transformer_block.TransformerBlock.ln2"]], "mlp (transformer_lens.components.transformer_block.transformerblock attribute)": [[33, "transformer_lens.components.transformer_block.TransformerBlock.mlp"]], "transformer_lens.components.transformer_block": [[33, "module-transformer_lens.components.transformer_block"]], "unembed (class in transformer_lens.components.unembed)": [[34, "transformer_lens.components.unembed.Unembed"]], "forward() (transformer_lens.components.unembed.unembed method)": [[34, "transformer_lens.components.unembed.Unembed.forward"]], "transformer_lens.components.unembed": [[34, "module-transformer_lens.components.unembed"]], "ioidataset (class in transformer_lens.evals)": [[35, "transformer_lens.evals.IOIDataset"]], "evaluate() (in module transformer_lens.evals)": [[35, "transformer_lens.evals.evaluate"]], "evaluate_on_dataset() (in module transformer_lens.evals)": [[35, "transformer_lens.evals.evaluate_on_dataset"]], "get_default_names() (transformer_lens.evals.ioidataset static method)": [[35, "transformer_lens.evals.IOIDataset.get_default_names"]], "get_default_nouns() (transformer_lens.evals.ioidataset static method)": [[35, "transformer_lens.evals.IOIDataset.get_default_nouns"]], "get_default_templates() (transformer_lens.evals.ioidataset static method)": [[35, "transformer_lens.evals.IOIDataset.get_default_templates"]], "get_sample() (transformer_lens.evals.ioidataset method)": [[35, "transformer_lens.evals.IOIDataset.get_sample"]], "induction_loss() (in module transformer_lens.evals)": [[35, "transformer_lens.evals.induction_loss"]], "ioi_eval() (in module transformer_lens.evals)": [[35, "transformer_lens.evals.ioi_eval"]], "make_code_data_loader() (in module transformer_lens.evals)": [[35, "transformer_lens.evals.make_code_data_loader"]], "make_owt_data_loader() (in module transformer_lens.evals)": [[35, "transformer_lens.evals.make_owt_data_loader"]], "make_pile_data_loader() (in module transformer_lens.evals)": [[35, "transformer_lens.evals.make_pile_data_loader"]], "make_wiki_data_loader() (in module transformer_lens.evals)": [[35, "transformer_lens.evals.make_wiki_data_loader"]], "sanity_check() (in module transformer_lens.evals)": [[35, "transformer_lens.evals.sanity_check"]], "transformer_lens.evals": [[35, "module-transformer_lens.evals"]], "compute_head_attention_similarity_score() (in module transformer_lens.head_detector)": [[36, "transformer_lens.head_detector.compute_head_attention_similarity_score"]], "detect_head() (in module transformer_lens.head_detector)": [[36, "transformer_lens.head_detector.detect_head"]], "get_duplicate_token_head_detection_pattern() (in module transformer_lens.head_detector)": [[36, "transformer_lens.head_detector.get_duplicate_token_head_detection_pattern"]], "get_induction_head_detection_pattern() (in module transformer_lens.head_detector)": [[36, "transformer_lens.head_detector.get_induction_head_detection_pattern"]], "get_previous_token_head_detection_pattern() (in module transformer_lens.head_detector)": [[36, "transformer_lens.head_detector.get_previous_token_head_detection_pattern"]], "get_supported_heads() (in module transformer_lens.head_detector)": [[36, "transformer_lens.head_detector.get_supported_heads"]], "transformer_lens.head_detector": [[36, "module-transformer_lens.head_detector"]], "hookfunction (in module transformer_lens.hook_points)": [[37, "transformer_lens.hook_points.HookFunction"]], "hookpoint (class in transformer_lens.hook_points)": [[37, "transformer_lens.hook_points.HookPoint"]], "hookedrootmodule (class in transformer_lens.hook_points)": [[37, "transformer_lens.hook_points.HookedRootModule"]], "lenshandle (class in transformer_lens.hook_points)": [[37, "transformer_lens.hook_points.LensHandle"]], "add_caching_hooks() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.add_caching_hooks"]], "add_hook() (transformer_lens.hook_points.hookpoint method)": [[37, "transformer_lens.hook_points.HookPoint.add_hook"]], "add_hook() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.add_hook"]], "add_perma_hook() (transformer_lens.hook_points.hookpoint method)": [[37, "transformer_lens.hook_points.HookPoint.add_perma_hook"]], "add_perma_hook() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.add_perma_hook"]], "cache_all() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.cache_all"]], "cache_some() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.cache_some"]], "check_and_add_hook() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.check_and_add_hook"]], "check_hooks_to_add() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.check_hooks_to_add"]], "clear_context() (transformer_lens.hook_points.hookpoint method)": [[37, "transformer_lens.hook_points.HookPoint.clear_context"]], "clear_contexts() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.clear_contexts"]], "context_level (transformer_lens.hook_points.lenshandle attribute)": [[37, "transformer_lens.hook_points.LensHandle.context_level"]], "forward() (transformer_lens.hook_points.hookpoint method)": [[37, "transformer_lens.hook_points.HookPoint.forward"]], "get_caching_hooks() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.get_caching_hooks"]], "hook (transformer_lens.hook_points.lenshandle attribute)": [[37, "transformer_lens.hook_points.LensHandle.hook"]], "hook_dict (transformer_lens.hook_points.hookedrootmodule attribute)": [[37, "transformer_lens.hook_points.HookedRootModule.hook_dict"]], "hook_points() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.hook_points"]], "hooks() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.hooks"]], "is_permanent (transformer_lens.hook_points.lenshandle attribute)": [[37, "transformer_lens.hook_points.LensHandle.is_permanent"]], "layer() (transformer_lens.hook_points.hookpoint method)": [[37, "transformer_lens.hook_points.HookPoint.layer"]], "mod_dict (transformer_lens.hook_points.hookedrootmodule attribute)": [[37, "transformer_lens.hook_points.HookedRootModule.mod_dict"]], "name (transformer_lens.hook_points.hookedrootmodule attribute)": [[37, "transformer_lens.hook_points.HookedRootModule.name"]], "remove_all_hook_fns() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.remove_all_hook_fns"]], "remove_hooks() (transformer_lens.hook_points.hookpoint method)": [[37, "transformer_lens.hook_points.HookPoint.remove_hooks"]], "reset_hooks() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.reset_hooks"]], "run_with_cache() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.run_with_cache"]], "run_with_hooks() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.run_with_hooks"]], "setup() (transformer_lens.hook_points.hookedrootmodule method)": [[37, "transformer_lens.hook_points.HookedRootModule.setup"]], "transformer_lens.hook_points": [[37, "module-transformer_lens.hook_points"]], "config (class in transformer_lens.loading_from_pretrained)": [[38, "transformer_lens.loading_from_pretrained.Config"]], "model_aliases (in module transformer_lens.loading_from_pretrained)": [[38, "transformer_lens.loading_from_pretrained.MODEL_ALIASES"]], "non_hf_hosted_model_names (in module transformer_lens.loading_from_pretrained)": [[38, "transformer_lens.loading_from_pretrained.NON_HF_HOSTED_MODEL_NAMES"]], "official_model_names (in module transformer_lens.loading_from_pretrained)": [[38, "transformer_lens.loading_from_pretrained.OFFICIAL_MODEL_NAMES"]], "d_head (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.d_head"]], "d_mlp (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.d_mlp"]], "d_model (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.d_model"]], "d_vocab (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.d_vocab"]], "debug (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.debug"]], "get_checkpoint_labels() (in module transformer_lens.loading_from_pretrained)": [[38, "transformer_lens.loading_from_pretrained.get_checkpoint_labels"]], "get_num_params_of_pretrained() (in module transformer_lens.loading_from_pretrained)": [[38, "transformer_lens.loading_from_pretrained.get_num_params_of_pretrained"]], "get_pretrained_model_config() (in module transformer_lens.loading_from_pretrained)": [[38, "transformer_lens.loading_from_pretrained.get_pretrained_model_config"]], "init_range (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.init_range"]], "layer_norm_eps (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.layer_norm_eps"]], "n_ctx (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.n_ctx"]], "n_heads (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.n_heads"]], "n_layers (transformer_lens.loading_from_pretrained.config attribute)": [[38, "transformer_lens.loading_from_pretrained.Config.n_layers"]], "transformer_lens.loading_from_pretrained": [[38, "module-transformer_lens.loading_from_pretrained"]], "hookedtransformerkeyvaluecache (class in transformer_lens.past_key_value_caching)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCache"]], "hookedtransformerkeyvaluecacheentry (class in transformer_lens.past_key_value_caching)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCacheEntry"]], "append() (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecacheentry method)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCacheEntry.append"]], "append_attention_mask() (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecache method)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCache.append_attention_mask"]], "entries (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecache attribute)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCache.entries"]], "freeze() (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecache method)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCache.freeze"]], "frozen (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecache attribute)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCache.frozen"]], "frozen (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecacheentry attribute)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCacheEntry.frozen"]], "init_cache() (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecache class method)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCache.init_cache"]], "init_cache_entry() (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecacheentry class method)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCacheEntry.init_cache_entry"]], "past_keys (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecacheentry attribute)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCacheEntry.past_keys"]], "past_values (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecacheentry attribute)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCacheEntry.past_values"]], "previous_attention_mask (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecache attribute)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCache.previous_attention_mask"]], "transformer_lens.past_key_value_caching": [[39, "module-transformer_lens.past_key_value_caching"]], "unfreeze() (transformer_lens.past_key_value_caching.hookedtransformerkeyvaluecache method)": [[39, "transformer_lens.past_key_value_caching.HookedTransformerKeyValueCache.unfreeze"]], "generic_activation_patch() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.generic_activation_patch"]], "get_act_patch_attn_head_all_pos_every() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_all_pos_every"]], "get_act_patch_attn_head_by_pos_every() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_by_pos_every"]], "get_act_patch_attn_head_k_all_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_k_all_pos"]], "get_act_patch_attn_head_k_by_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_k_by_pos"]], "get_act_patch_attn_head_out_all_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_out_all_pos"]], "get_act_patch_attn_head_out_by_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_out_by_pos"]], "get_act_patch_attn_head_pattern_all_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_pattern_all_pos"]], "get_act_patch_attn_head_pattern_by_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_pattern_by_pos"]], "get_act_patch_attn_head_pattern_dest_src_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_pattern_dest_src_pos"]], "get_act_patch_attn_head_q_all_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_q_all_pos"]], "get_act_patch_attn_head_q_by_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_q_by_pos"]], "get_act_patch_attn_head_v_all_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_v_all_pos"]], "get_act_patch_attn_head_v_by_pos() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_head_v_by_pos"]], "get_act_patch_attn_out() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_attn_out"]], "get_act_patch_block_every() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_block_every"]], "get_act_patch_mlp_out() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_mlp_out"]], "get_act_patch_resid_mid() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_resid_mid"]], "get_act_patch_resid_pre() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.get_act_patch_resid_pre"]], "layer_head_dest_src_pos_pattern_patch_setter() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.layer_head_dest_src_pos_pattern_patch_setter"]], "layer_head_pattern_patch_setter() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.layer_head_pattern_patch_setter"]], "layer_head_pos_pattern_patch_setter() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.layer_head_pos_pattern_patch_setter"]], "layer_head_vector_patch_setter() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.layer_head_vector_patch_setter"]], "layer_pos_head_vector_patch_setter() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.layer_pos_head_vector_patch_setter"]], "layer_pos_patch_setter() (in module transformer_lens.patching)": [[40, "transformer_lens.patching.layer_pos_patch_setter"]], "transformer_lens.patching": [[40, "module-transformer_lens.patching"]], "transformer_lens.pretrained.weight_conversions.bert": [[43, "module-transformer_lens.pretrained.weight_conversions.bert"]], "convert_bloom_weights() (in module transformer_lens.pretrained.weight_conversions.bloom)": [[44, "transformer_lens.pretrained.weight_conversions.bloom.convert_bloom_weights"]], "transformer_lens.pretrained.weight_conversions.bloom": [[44, "module-transformer_lens.pretrained.weight_conversions.bloom"]], "convert_coder_weights() (in module transformer_lens.pretrained.weight_conversions.coder)": [[45, "transformer_lens.pretrained.weight_conversions.coder.convert_coder_weights"]], "transformer_lens.pretrained.weight_conversions.coder": [[45, "module-transformer_lens.pretrained.weight_conversions.coder"]], "transformer_lens.pretrained.weight_conversions.gemma": [[46, "module-transformer_lens.pretrained.weight_conversions.gemma"]], "transformer_lens.pretrained.weight_conversions.gpt2": [[47, "module-transformer_lens.pretrained.weight_conversions.gpt2"]], "transformer_lens.pretrained.weight_conversions.gptj": [[48, "module-transformer_lens.pretrained.weight_conversions.gptj"]], "transformer_lens.pretrained.weight_conversions.llama": [[49, "module-transformer_lens.pretrained.weight_conversions.llama"]], "transformer_lens.pretrained.weight_conversions.mingpt": [[50, "module-transformer_lens.pretrained.weight_conversions.mingpt"]], "convert_mistral_weights() (in module transformer_lens.pretrained.weight_conversions.mistral)": [[51, "transformer_lens.pretrained.weight_conversions.mistral.convert_mistral_weights"]], "transformer_lens.pretrained.weight_conversions.mistral": [[51, "module-transformer_lens.pretrained.weight_conversions.mistral"]], "convert_mixtral_weights() (in module transformer_lens.pretrained.weight_conversions.mixtral)": [[52, "transformer_lens.pretrained.weight_conversions.mixtral.convert_mixtral_weights"]], "transformer_lens.pretrained.weight_conversions.mixtral": [[52, "module-transformer_lens.pretrained.weight_conversions.mixtral"]], "transformer_lens.pretrained.weight_conversions.nanogpt": [[53, "module-transformer_lens.pretrained.weight_conversions.nanogpt"]], "transformer_lens.pretrained.weight_conversions.neel_solu_old": [[54, "module-transformer_lens.pretrained.weight_conversions.neel_solu_old"]], "transformer_lens.pretrained.weight_conversions.neo": [[55, "module-transformer_lens.pretrained.weight_conversions.neo"]], "transformer_lens.pretrained.weight_conversions.neox": [[56, "module-transformer_lens.pretrained.weight_conversions.neox"]], "transformer_lens.pretrained.weight_conversions.opt": [[57, "module-transformer_lens.pretrained.weight_conversions.opt"]], "convert_phi_weights() (in module transformer_lens.pretrained.weight_conversions.phi)": [[58, "transformer_lens.pretrained.weight_conversions.phi.convert_phi_weights"]], "transformer_lens.pretrained.weight_conversions.phi": [[58, "module-transformer_lens.pretrained.weight_conversions.phi"]], "convert_phi3_weights() (in module transformer_lens.pretrained.weight_conversions.phi3)": [[59, "transformer_lens.pretrained.weight_conversions.phi3.convert_phi3_weights"]], "transformer_lens.pretrained.weight_conversions.phi3": [[59, "module-transformer_lens.pretrained.weight_conversions.phi3"]], "convert_qwen_weights() (in module transformer_lens.pretrained.weight_conversions.qwen)": [[60, "transformer_lens.pretrained.weight_conversions.qwen.convert_qwen_weights"]], "transformer_lens.pretrained.weight_conversions.qwen": [[60, "module-transformer_lens.pretrained.weight_conversions.qwen"]], "convert_qwen2_weights() (in module transformer_lens.pretrained.weight_conversions.qwen2)": [[61, "transformer_lens.pretrained.weight_conversions.qwen2.convert_qwen2_weights"]], "transformer_lens.pretrained.weight_conversions.qwen2": [[61, "module-transformer_lens.pretrained.weight_conversions.qwen2"]], "convert_t5_weights() (in module transformer_lens.pretrained.weight_conversions.t5)": [[62, "transformer_lens.pretrained.weight_conversions.t5.convert_t5_weights"]], "transformer_lens.pretrained.weight_conversions.t5": [[62, "module-transformer_lens.pretrained.weight_conversions.t5"]], "hookedtransformertrainconfig (class in transformer_lens.train)": [[63, "transformer_lens.train.HookedTransformerTrainConfig"]], "batch_size (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.batch_size"]], "device (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.device"]], "lr (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.lr"]], "max_grad_norm (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.max_grad_norm"]], "max_steps (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.max_steps"]], "momentum (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.momentum"]], "num_epochs (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.num_epochs"]], "optimizer_name (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.optimizer_name"]], "print_every (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.print_every"]], "save_dir (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.save_dir"]], "save_every (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.save_every"]], "seed (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.seed"]], "train() (in module transformer_lens.train)": [[63, "transformer_lens.train.train"]], "transformer_lens.train": [[63, "module-transformer_lens.train"]], "wandb (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.wandb"]], "wandb_project_name (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.wandb_project_name"]], "warmup_steps (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.warmup_steps"]], "weight_decay (transformer_lens.train.hookedtransformertrainconfig attribute)": [[63, "transformer_lens.train.HookedTransformerTrainConfig.weight_decay"]], "transformer_lens.utilities.activation_functions": [[65, "module-transformer_lens.utilities.activation_functions"]], "batch_addmm() (in module transformer_lens.utilities.addmm)": [[66, "transformer_lens.utilities.addmm.batch_addmm"]], "transformer_lens.utilities.addmm": [[66, "module-transformer_lens.utilities.addmm"]], "vanilla_addmm() (in module transformer_lens.utilities.addmm)": [[66, "transformer_lens.utilities.addmm.vanilla_addmm"]], "complex_attn_linear() (in module transformer_lens.utilities.attention)": [[67, "transformer_lens.utilities.attention.complex_attn_linear"]], "simple_attn_linear() (in module transformer_lens.utilities.attention)": [[67, "transformer_lens.utilities.attention.simple_attn_linear"]], "transformer_lens.utilities.attention": [[67, "module-transformer_lens.utilities.attention"]], "get_device_for_block_index() (in module transformer_lens.utilities.devices)": [[68, "transformer_lens.utilities.devices.get_device_for_block_index"]], "move_to_and_update_config() (in module transformer_lens.utilities.devices)": [[68, "transformer_lens.utilities.devices.move_to_and_update_config"]], "transformer_lens.utilities.devices": [[68, "module-transformer_lens.utilities.devices"]], "locallyoverridendefaults (class in transformer_lens.utils)": [[69, "transformer_lens.utils.LocallyOverridenDefaults"]], "slice (class in transformer_lens.utils)": [[69, "transformer_lens.utils.Slice"]], "sliceinput (in module transformer_lens.utils)": [[69, "transformer_lens.utils.SliceInput"]], "__init__() (transformer_lens.utils.locallyoverridendefaults method)": [[69, "transformer_lens.utils.LocallyOverridenDefaults.__init__"]], "__init__() (transformer_lens.utils.slice method)": [[69, "transformer_lens.utils.Slice.__init__"]], "apply() (transformer_lens.utils.slice method)": [[69, "transformer_lens.utils.Slice.apply"]], "calc_fan_in_and_fan_out() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.calc_fan_in_and_fan_out"]], "composition_scores() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.composition_scores"]], "download_file_from_hf() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.download_file_from_hf"]], "gelu_fast() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.gelu_fast"]], "gelu_new() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.gelu_new"]], "get_act_name() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_act_name"]], "get_attention_mask() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_attention_mask"]], "get_corner() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_corner"]], "get_cumsum_along_dim() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_cumsum_along_dim"]], "get_dataset() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_dataset"]], "get_device() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_device"]], "get_input_with_manually_prepended_bos() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_input_with_manually_prepended_bos"]], "get_nested_attr() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_nested_attr"]], "get_offset_position_ids() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_offset_position_ids"]], "get_tokenizer_with_bos() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_tokenizer_with_bos"]], "get_tokens_with_bos_removed() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.get_tokens_with_bos_removed"]], "indices() (transformer_lens.utils.slice method)": [[69, "transformer_lens.utils.Slice.indices"]], "init_kaiming_normal_() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.init_kaiming_normal_"]], "init_kaiming_uniform_() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.init_kaiming_uniform_"]], "init_xavier_normal_() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.init_xavier_normal_"]], "init_xavier_uniform_() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.init_xavier_uniform_"]], "is_lower_triangular() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.is_lower_triangular"]], "is_square() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.is_square"]], "keep_single_column() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.keep_single_column"]], "lm_accuracy() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.lm_accuracy"]], "lm_cross_entropy_loss() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.lm_cross_entropy_loss"]], "override_or_use_default_value() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.override_or_use_default_value"]], "print_gpu_mem() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.print_gpu_mem"]], "remove_batch_dim() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.remove_batch_dim"]], "repeat_along_head_dimension() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.repeat_along_head_dimension"]], "sample_logits() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.sample_logits"]], "set_nested_attr() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.set_nested_attr"]], "slice (transformer_lens.utils.slice attribute)": [[69, "transformer_lens.utils.Slice.slice"]], "solu() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.solu"]], "test_prompt() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.test_prompt"]], "to_numpy() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.to_numpy"]], "tokenize_and_concatenate() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.tokenize_and_concatenate"]], "transformer_lens.utils": [[69, "module-transformer_lens.utils"]], "transpose() (in module transformer_lens.utils)": [[69, "transformer_lens.utils.transpose"]], "unwrap() (transformer_lens.utils.slice class method)": [[69, "transformer_lens.utils.Slice.unwrap"]]}})
\ No newline at end of file