diff --git a/1-preparing-to-train.html b/1-preparing-to-train.html
index c39f99ce..1d896c88 100644
--- a/1-preparing-to-train.html
+++ b/1-preparing-to-train.html
@@ -472,7 +472,7 @@ <h3 class="callout-title">Challenge<a class="anchor" aria-label="anchor" href="#
 <button class="accordion-button solution-button collapsed" type="button" data-bs-toggle="collapse" data-bs-target="#collapseSolution1" aria-expanded="false" aria-controls="collapseSolution1">
   <h4 class="accordion-header" id="headingSolution1"> Show me the solution </h4>
 </button>
-<div id="collapseSolution1" class="accordion-collapse collapse" aria-labelledby="headingSolution1" data-bs-parent="#accordionSolution1">
+<div id="collapseSolution1" class="accordion-collapse collapse" data-bs-parent="#accordionSolution1" aria-labelledby="headingSolution1">
 <div class="accordion-body">
 <p>A summary of the principles is listed below:</p>
 <ul><li>Social and clinical value: Does the social or clinical value of
diff --git a/5a-explainable-AI-method-overview.html b/5a-explainable-AI-method-overview.html
index 7b56c255..6838f420 100644
--- a/5a-explainable-AI-method-overview.html
+++ b/5a-explainable-AI-method-overview.html
@@ -691,7 +691,7 @@ <h3 class="callout-title">Classifying explanation techniques<a class="anchor" ar
 <button class="accordion-button solution-button collapsed" type="button" data-bs-toggle="collapse" data-bs-target="#collapseSolution1" aria-expanded="false" aria-controls="collapseSolution1">
   <h4 class="accordion-header" id="headingSolution1"> Show me the solution </h4>
 </button>
-<div id="collapseSolution1" class="accordion-collapse collapse" data-bs-parent="#accordionSolution1" aria-labelledby="headingSolution1">
+<div id="collapseSolution1" class="accordion-collapse collapse" aria-labelledby="headingSolution1" data-bs-parent="#accordionSolution1">
 <div class="accordion-body">
 <table class="table"><colgroup><col width="54%"><col width="21%"><col width="10%"><col width="13%"></colgroup><thead><tr class="header"><th>Approach</th>
 <th>Post Hoc or Inherently Interpretable?</th>
diff --git a/7a-OOD-detection-output-based.html b/7a-OOD-detection-output-based.html
index 1d19b921..7112df98 100644
--- a/7a-OOD-detection-output-based.html
+++ b/7a-OOD-detection-output-based.html
@@ -412,8 +412,8 @@ <h2 class="card-header">Overview</h2>
 <h3 class="card-title">Questions</h3>
 <ul><li>What are out-of-distribution (OOD) data and why is detecting them
 important in machine learning models?</li>
-<li>How do output-based methods like softmax, energy-based, and
-distance-based methods work for OOD detection?</li>
+<li>How do output-based methods like softmax and energy-based methods
+work for OOD detection?</li>
 <li>What are the limitations of output-based OOD detection methods?</li>
 </ul></div>
 </div>
@@ -988,7 +988,7 @@ <h3 class="code-label">PYTHON<i aria-hidden="true" data-feather="chevron-left"><
 <span id="cb14-29"><a href="#cb14-29" tabindex="-1"></a>  all_true_labels <span class="op">=</span> np.concatenate([<span class="op">-</span><span class="dv">1</span> <span class="op">*</span> np.ones(ood_classifications.shape), train_labels])</span>
 <span id="cb14-30"><a href="#cb14-30" tabindex="-1"></a></span>
 <span id="cb14-31"><a href="#cb14-31" tabindex="-1"></a>  <span class="co"># Evaluate metrics</span></span>
-<span id="cb14-32"><a href="#cb14-32" tabindex="-1"></a>  precision, recall, f1, _ <span class="op">=</span> precision_recall_fscore_support(all_true_labels, all_predictions, labels<span class="op">=</span>[<span class="dv">0</span>, <span class="dv">1</span>], average<span class="op">=</span><span class="st">'macro'</span>) <span class="co"># discuss macro vs micro .</span></span>
+<span id="cb14-32"><a href="#cb14-32" tabindex="-1"></a>  precision, recall, f1, _ <span class="op">=</span> precision_recall_fscore_support(all_true_labels, all_predictions, labels<span class="op">=</span>[<span class="dv">0</span>, <span class="dv">1</span>, <span class="op">-</span><span class="dv">1</span>], average<span class="op">=</span><span class="st">'macro'</span>) <span class="co"># discuss macro vs micro .</span></span>
 <span id="cb14-33"><a href="#cb14-33" tabindex="-1"></a>  accuracy <span class="op">=</span> accuracy_score(all_true_labels, all_predictions)</span>
 <span id="cb14-34"><a href="#cb14-34" tabindex="-1"></a></span>
 <span id="cb14-35"><a href="#cb14-35" tabindex="-1"></a>  accuracies.append(accuracy)</span>
@@ -1027,13 +1027,51 @@ <h3 class="code-label">PYTHON<i aria-hidden="true" data-feather="chevron-left"><
 <span id="cb14-68"><a href="#cb14-68" tabindex="-1"></a>plt.legend()</span>
 <span id="cb14-69"><a href="#cb14-69" tabindex="-1"></a>plt.show()</span></code></pre>
 </div>
+<div class="codewrapper sourceCode" id="cb15">
+<h3 class="code-label">PYTHON<i aria-hidden="true" data-feather="chevron-left"></i><i aria-hidden="true" data-feather="chevron-right"></i>
+</h3>
+<pre class="sourceCode python" tabindex="0"><code class="sourceCode python"><span id="cb15-1"><a href="#cb15-1" tabindex="-1"></a><span class="im">import</span> numpy <span class="im">as</span> np</span>
+<span id="cb15-2"><a href="#cb15-2" tabindex="-1"></a><span class="im">import</span> matplotlib.pyplot <span class="im">as</span> plt</span>
+<span id="cb15-3"><a href="#cb15-3" tabindex="-1"></a><span class="im">from</span> sklearn.metrics <span class="im">import</span> confusion_matrix, ConfusionMatrixDisplay</span>
+<span id="cb15-4"><a href="#cb15-4" tabindex="-1"></a></span>
+<span id="cb15-5"><a href="#cb15-5" tabindex="-1"></a><span class="co"># Assuming ood_probs, id_probs, and train_labels are defined</span></span>
+<span id="cb15-6"><a href="#cb15-6" tabindex="-1"></a><span class="co"># Threshold values</span></span>
+<span id="cb15-7"><a href="#cb15-7" tabindex="-1"></a>upper_threshold <span class="op">=</span> best_f1_threshold</span>
+<span id="cb15-8"><a href="#cb15-8" tabindex="-1"></a></span>
+<span id="cb15-9"><a href="#cb15-9" tabindex="-1"></a><span class="co"># Classifying OOD examples (sandals)</span></span>
+<span id="cb15-10"><a href="#cb15-10" tabindex="-1"></a>ood_classifications <span class="op">=</span> np.where(ood_probs[:, <span class="dv">1</span>] <span class="op">&gt;=</span> upper_threshold, <span class="dv">1</span>,  <span class="co"># classified as pants</span></span>
+<span id="cb15-11"><a href="#cb15-11" tabindex="-1"></a>                               np.where(ood_probs[:, <span class="dv">0</span>] <span class="op">&gt;=</span> upper_threshold, <span class="dv">0</span>,  <span class="co"># classified as shirts</span></span>
+<span id="cb15-12"><a href="#cb15-12" tabindex="-1"></a>                                        <span class="op">-</span><span class="dv">1</span>))  <span class="co"># classified as OOD</span></span>
+<span id="cb15-13"><a href="#cb15-13" tabindex="-1"></a>ood_classifications</span>
+<span id="cb15-14"><a href="#cb15-14" tabindex="-1"></a></span>
+<span id="cb15-15"><a href="#cb15-15" tabindex="-1"></a>id_probs</span>
+<span id="cb15-16"><a href="#cb15-16" tabindex="-1"></a><span class="co"># Classifying ID examples (T-shirts and pants)</span></span>
+<span id="cb15-17"><a href="#cb15-17" tabindex="-1"></a>id_classifications <span class="op">=</span> np.where(id_probs[:, <span class="dv">1</span>] <span class="op">&gt;=</span> upper_threshold, <span class="dv">1</span>,  <span class="co"># classified as pants</span></span>
+<span id="cb15-18"><a href="#cb15-18" tabindex="-1"></a>                              np.where(id_probs[:, <span class="dv">0</span>] <span class="op">&gt;=</span> upper_threshold, <span class="dv">0</span>,  <span class="co"># classified as shirts</span></span>
+<span id="cb15-19"><a href="#cb15-19" tabindex="-1"></a>                                       <span class="op">-</span><span class="dv">1</span>))  <span class="co"># classified as OOD</span></span>
+<span id="cb15-20"><a href="#cb15-20" tabindex="-1"></a></span>
+<span id="cb15-21"><a href="#cb15-21" tabindex="-1"></a>id_classifications</span>
+<span id="cb15-22"><a href="#cb15-22" tabindex="-1"></a></span>
+<span id="cb15-23"><a href="#cb15-23" tabindex="-1"></a><span class="co"># Combine OOD and ID classifications and true labels</span></span>
+<span id="cb15-24"><a href="#cb15-24" tabindex="-1"></a>all_predictions <span class="op">=</span> np.concatenate([ood_classifications, id_classifications])</span>
+<span id="cb15-25"><a href="#cb15-25" tabindex="-1"></a>all_true_labels <span class="op">=</span> np.concatenate([<span class="op">-</span><span class="dv">1</span> <span class="op">*</span> np.ones(ood_classifications.shape), train_labels])</span>
+<span id="cb15-26"><a href="#cb15-26" tabindex="-1"></a></span>
+<span id="cb15-27"><a href="#cb15-27" tabindex="-1"></a><span class="co"># Confusion matrix</span></span>
+<span id="cb15-28"><a href="#cb15-28" tabindex="-1"></a>cm <span class="op">=</span> confusion_matrix(all_true_labels, all_predictions, labels<span class="op">=</span>[<span class="dv">0</span>, <span class="dv">1</span>, <span class="op">-</span><span class="dv">1</span>])</span>
+<span id="cb15-29"><a href="#cb15-29" tabindex="-1"></a></span>
+<span id="cb15-30"><a href="#cb15-30" tabindex="-1"></a><span class="co"># Plotting the confusion matrix</span></span>
+<span id="cb15-31"><a href="#cb15-31" tabindex="-1"></a>disp <span class="op">=</span> ConfusionMatrixDisplay(confusion_matrix<span class="op">=</span>cm, display_labels<span class="op">=</span>[<span class="st">"Shirt"</span>, <span class="st">"Pants"</span>, <span class="st">"OOD"</span>])</span>
+<span id="cb15-32"><a href="#cb15-32" tabindex="-1"></a>disp.plot(cmap<span class="op">=</span>plt.cm.Blues)</span>
+<span id="cb15-33"><a href="#cb15-33" tabindex="-1"></a>plt.title(<span class="st">'Confusion Matrix for OOD and ID Classification'</span>)</span>
+<span id="cb15-34"><a href="#cb15-34" tabindex="-1"></a>plt.show()</span></code></pre>
+</div>
 </div>
 </div>
 </div>
 <div class="section level1">
 <h1 id="example-2-energy-based-ood-detection">Example 2: Energy-Based OOD Detection<a class="anchor" aria-label="anchor" href="#example-2-energy-based-ood-detection"></a></h1>
-<p>Liu et al., Energy-based Out-of-distribution Detection, NeurIPS
-2020</p>
+<p>Liu et al., Energy-based Out-of-distribution Detection, NeurIPS 2020;
+<a href="https://arxiv.org/pdf/2010.03759" class="external-link uri">https://arxiv.org/pdf/2010.03759</a></p>
 <ul><li><p>E(x, y) = energy value</p></li>
 <li><p>if x and y are “compatitble”, lower energy</p></li>
 <li>
@@ -1043,16 +1081,369 @@ <h1 id="example-2-energy-based-ood-detection">Example 2: Energy-Based OOD Detect
 </ul></li>
 <li><p>With energy scores, ID and OOD distributions become much more
 separable</p></li>
-<li><p>Another “output-based” method like softmax # Conclusion</p></li>
-</ul><div class="codewrapper sourceCode" id="cb15">
+<li><p>Another “output-based” method like softmax</p></li>
+</ul><div class="section level2">
+<h2 id="pytorch-out-of-distribution-detection">PyTorch Out-of-Distribution Detection<a class="anchor" aria-label="anchor" href="#pytorch-out-of-distribution-detection"></a></h2>
+<p>There’s a Pytorch package for OOD detection! <a href="https://pytorch-ood.readthedocs.io/en/latest/info.html" class="external-link uri">https://pytorch-ood.readthedocs.io/en/latest/info.html</a></p>
+<div class="codewrapper sourceCode" id="cb16">
+<h3 class="code-label">PYTHON<i aria-hidden="true" data-feather="chevron-left"></i><i aria-hidden="true" data-feather="chevron-right"></i>
+</h3>
+<pre class="sourceCode python" tabindex="0"><code class="sourceCode python"><span id="cb16-1"><a href="#cb16-1" tabindex="-1"></a><span class="op">!</span>pip install pytorch<span class="op">-</span>ood</span></code></pre>
+</div>
+<div class="section level3">
+<h3 id="energy-based-is-designed-to-work-with-neural-nets-unpack-this-">Energy-based is designed to work with neural nets… unpack this.<a class="anchor" aria-label="anchor" href="#energy-based-is-designed-to-work-with-neural-nets-unpack-this-"></a></h3>
+<p>Let’s train a simple CNN model on the FashionMNIST dataset.</p>
+<div class="codewrapper sourceCode" id="cb17">
+<h3 class="code-label">PYTHON<i aria-hidden="true" data-feather="chevron-left"></i><i aria-hidden="true" data-feather="chevron-right"></i>
+</h3>
+<pre class="sourceCode python" tabindex="0"><code class="sourceCode python"><span id="cb17-1"><a href="#cb17-1" tabindex="-1"></a><span class="im">import</span> torch</span>
+<span id="cb17-2"><a href="#cb17-2" tabindex="-1"></a><span class="im">import</span> torch.nn <span class="im">as</span> nn</span>
+<span id="cb17-3"><a href="#cb17-3" tabindex="-1"></a><span class="im">import</span> torch.optim <span class="im">as</span> optim</span>
+<span id="cb17-4"><a href="#cb17-4" tabindex="-1"></a><span class="im">import</span> torchvision.transforms <span class="im">as</span> transforms</span>
+<span id="cb17-5"><a href="#cb17-5" tabindex="-1"></a><span class="im">from</span> keras.datasets <span class="im">import</span> fashion_mnist</span>
+<span id="cb17-6"><a href="#cb17-6" tabindex="-1"></a><span class="im">import</span> torch.nn.functional <span class="im">as</span> F</span>
+<span id="cb17-7"><a href="#cb17-7" tabindex="-1"></a><span class="im">import</span> matplotlib.pyplot <span class="im">as</span> plt</span>
+<span id="cb17-8"><a href="#cb17-8" tabindex="-1"></a><span class="im">import</span> numpy <span class="im">as</span> np</span>
+<span id="cb17-9"><a href="#cb17-9" tabindex="-1"></a></span>
+<span id="cb17-10"><a href="#cb17-10" tabindex="-1"></a><span class="co"># Load Fashion MNIST dataset</span></span>
+<span id="cb17-11"><a href="#cb17-11" tabindex="-1"></a>(train_images, train_labels), (test_images, test_labels) <span class="op">=</span> fashion_mnist.load_data()</span>
+<span id="cb17-12"><a href="#cb17-12" tabindex="-1"></a></span>
+<span id="cb17-13"><a href="#cb17-13" tabindex="-1"></a><span class="co"># Define classes for simplicity</span></span>
+<span id="cb17-14"><a href="#cb17-14" tabindex="-1"></a>class_names <span class="op">=</span> [<span class="st">'T-shirt/top'</span>, <span class="st">'Trouser'</span>, <span class="st">'Pullover'</span>, <span class="st">'Dress'</span>, <span class="st">'Coat'</span>,</span>
+<span id="cb17-15"><a href="#cb17-15" tabindex="-1"></a>               <span class="st">'Sandal'</span>, <span class="st">'Shirt'</span>, <span class="st">'Sneaker'</span>, <span class="st">'Bag'</span>, <span class="st">'Ankle boot'</span>]</span>
+<span id="cb17-16"><a href="#cb17-16" tabindex="-1"></a></span>
+<span id="cb17-17"><a href="#cb17-17" tabindex="-1"></a><span class="co"># Prepare OOD data - Sandals (5)</span></span>
+<span id="cb17-18"><a href="#cb17-18" tabindex="-1"></a>ood_data <span class="op">=</span> test_images[test_labels <span class="op">==</span> <span class="dv">5</span>]</span>
+<span id="cb17-19"><a href="#cb17-19" tabindex="-1"></a>ood_labels <span class="op">=</span> test_labels[test_labels <span class="op">==</span> <span class="dv">5</span>]</span>
+<span id="cb17-20"><a href="#cb17-20" tabindex="-1"></a><span class="bu">print</span>(<span class="ss">f'ood_data.shape=</span><span class="sc">{</span>ood_data<span class="sc">.</span>shape<span class="sc">}</span><span class="ss">'</span>)</span>
+<span id="cb17-21"><a href="#cb17-21" tabindex="-1"></a></span>
+<span id="cb17-22"><a href="#cb17-22" tabindex="-1"></a><span class="co"># Filter data for T-shirts (0) and Trousers (1) as in-distribution</span></span>
+<span id="cb17-23"><a href="#cb17-23" tabindex="-1"></a>train_filter <span class="op">=</span> np.isin(train_labels, [<span class="dv">0</span>, <span class="dv">1</span>])</span>
+<span id="cb17-24"><a href="#cb17-24" tabindex="-1"></a>test_filter <span class="op">=</span> np.isin(test_labels, [<span class="dv">0</span>, <span class="dv">1</span>])</span>
+<span id="cb17-25"><a href="#cb17-25" tabindex="-1"></a></span>
+<span id="cb17-26"><a href="#cb17-26" tabindex="-1"></a>train_data <span class="op">=</span> train_images[train_filter]</span>
+<span id="cb17-27"><a href="#cb17-27" tabindex="-1"></a>train_labels <span class="op">=</span> train_labels[train_filter]</span>
+<span id="cb17-28"><a href="#cb17-28" tabindex="-1"></a><span class="bu">print</span>(<span class="ss">f'train_data.shape=</span><span class="sc">{</span>train_data<span class="sc">.</span>shape<span class="sc">}</span><span class="ss">'</span>)</span>
+<span id="cb17-29"><a href="#cb17-29" tabindex="-1"></a></span>
+<span id="cb17-30"><a href="#cb17-30" tabindex="-1"></a>test_data <span class="op">=</span> test_images[test_filter]</span>
+<span id="cb17-31"><a href="#cb17-31" tabindex="-1"></a>test_labels <span class="op">=</span> test_labels[test_filter]</span>
+<span id="cb17-32"><a href="#cb17-32" tabindex="-1"></a><span class="bu">print</span>(<span class="ss">f'test_data.shape=</span><span class="sc">{</span>test_data<span class="sc">.</span>shape<span class="sc">}</span><span class="ss">'</span>)</span>
+<span id="cb17-33"><a href="#cb17-33" tabindex="-1"></a></span>
+<span id="cb17-34"><a href="#cb17-34" tabindex="-1"></a><span class="co"># Transform to Tensor and normalize</span></span>
+<span id="cb17-35"><a href="#cb17-35" tabindex="-1"></a>transform <span class="op">=</span> transforms.Compose([</span>
+<span id="cb17-36"><a href="#cb17-36" tabindex="-1"></a>    transforms.ToTensor(),</span>
+<span id="cb17-37"><a href="#cb17-37" tabindex="-1"></a>    transforms.Normalize((<span class="fl">0.5</span>,), (<span class="fl">0.5</span>,))</span>
+<span id="cb17-38"><a href="#cb17-38" tabindex="-1"></a>])</span>
+<span id="cb17-39"><a href="#cb17-39" tabindex="-1"></a></span>
+<span id="cb17-40"><a href="#cb17-40" tabindex="-1"></a><span class="co"># Convert to PyTorch tensors and normalize</span></span>
+<span id="cb17-41"><a href="#cb17-41" tabindex="-1"></a>train_data_tensor <span class="op">=</span> torch.tensor(train_data, dtype<span class="op">=</span>torch.float32).unsqueeze(<span class="dv">1</span>) <span class="op">/</span> <span class="fl">255.0</span></span>
+<span id="cb17-42"><a href="#cb17-42" tabindex="-1"></a>test_data_tensor <span class="op">=</span> torch.tensor(test_data, dtype<span class="op">=</span>torch.float32).unsqueeze(<span class="dv">1</span>) <span class="op">/</span> <span class="fl">255.0</span></span>
+<span id="cb17-43"><a href="#cb17-43" tabindex="-1"></a>ood_data_tensor <span class="op">=</span> torch.tensor(ood_data, dtype<span class="op">=</span>torch.float32).unsqueeze(<span class="dv">1</span>) <span class="op">/</span> <span class="fl">255.0</span></span>
+<span id="cb17-44"><a href="#cb17-44" tabindex="-1"></a></span>
+<span id="cb17-45"><a href="#cb17-45" tabindex="-1"></a>train_labels_tensor <span class="op">=</span> torch.tensor(train_labels, dtype<span class="op">=</span>torch.<span class="bu">long</span>)</span>
+<span id="cb17-46"><a href="#cb17-46" tabindex="-1"></a>test_labels_tensor <span class="op">=</span> torch.tensor(test_labels, dtype<span class="op">=</span>torch.<span class="bu">long</span>)</span>
+<span id="cb17-47"><a href="#cb17-47" tabindex="-1"></a></span>
+<span id="cb17-48"><a href="#cb17-48" tabindex="-1"></a>train_dataset <span class="op">=</span> torch.utils.data.TensorDataset(train_data_tensor, train_labels_tensor)</span>
+<span id="cb17-49"><a href="#cb17-49" tabindex="-1"></a>test_dataset <span class="op">=</span> torch.utils.data.TensorDataset(test_data_tensor, test_labels_tensor)</span>
+<span id="cb17-50"><a href="#cb17-50" tabindex="-1"></a>ood_dataset <span class="op">=</span> torch.utils.data.TensorDataset(ood_data_tensor, torch.zeros(ood_data_tensor.shape[<span class="dv">0</span>], dtype<span class="op">=</span>torch.<span class="bu">long</span>))</span>
+<span id="cb17-51"><a href="#cb17-51" tabindex="-1"></a></span>
+<span id="cb17-52"><a href="#cb17-52" tabindex="-1"></a>train_loader <span class="op">=</span> torch.utils.data.DataLoader(train_dataset, batch_size<span class="op">=</span><span class="dv">64</span>, shuffle<span class="op">=</span><span class="va">True</span>)</span>
+<span id="cb17-53"><a href="#cb17-53" tabindex="-1"></a>test_loader <span class="op">=</span> torch.utils.data.DataLoader(test_dataset, batch_size<span class="op">=</span><span class="dv">64</span>, shuffle<span class="op">=</span><span class="va">False</span>)</span>
+<span id="cb17-54"><a href="#cb17-54" tabindex="-1"></a>ood_loader <span class="op">=</span> torch.utils.data.DataLoader(ood_dataset, batch_size<span class="op">=</span><span class="dv">64</span>, shuffle<span class="op">=</span><span class="va">False</span>)</span>
+<span id="cb17-55"><a href="#cb17-55" tabindex="-1"></a></span>
+<span id="cb17-56"><a href="#cb17-56" tabindex="-1"></a><span class="co"># Define a simple CNN model</span></span>
+<span id="cb17-57"><a href="#cb17-57" tabindex="-1"></a><span class="kw">class</span> SimpleCNN(nn.Module):</span>
+<span id="cb17-58"><a href="#cb17-58" tabindex="-1"></a>    <span class="kw">def</span> <span class="fu">__init__</span>(<span class="va">self</span>):</span>
+<span id="cb17-59"><a href="#cb17-59" tabindex="-1"></a>        <span class="bu">super</span>(SimpleCNN, <span class="va">self</span>).<span class="fu">__init__</span>()</span>
+<span id="cb17-60"><a href="#cb17-60" tabindex="-1"></a>        <span class="va">self</span>.conv1 <span class="op">=</span> nn.Conv2d(<span class="dv">1</span>, <span class="dv">32</span>, kernel_size<span class="op">=</span><span class="dv">3</span>)</span>
+<span id="cb17-61"><a href="#cb17-61" tabindex="-1"></a>        <span class="va">self</span>.conv2 <span class="op">=</span> nn.Conv2d(<span class="dv">32</span>, <span class="dv">64</span>, kernel_size<span class="op">=</span><span class="dv">3</span>)</span>
+<span id="cb17-62"><a href="#cb17-62" tabindex="-1"></a>        <span class="va">self</span>.fc1 <span class="op">=</span> nn.Linear(<span class="dv">64</span><span class="op">*</span><span class="dv">5</span><span class="op">*</span><span class="dv">5</span>, <span class="dv">128</span>)  <span class="co"># Updated this line</span></span>
+<span id="cb17-63"><a href="#cb17-63" tabindex="-1"></a>        <span class="va">self</span>.fc2 <span class="op">=</span> nn.Linear(<span class="dv">128</span>, <span class="dv">2</span>)</span>
+<span id="cb17-64"><a href="#cb17-64" tabindex="-1"></a></span>
+<span id="cb17-65"><a href="#cb17-65" tabindex="-1"></a>    <span class="kw">def</span> forward(<span class="va">self</span>, x):</span>
+<span id="cb17-66"><a href="#cb17-66" tabindex="-1"></a>        x <span class="op">=</span> F.relu(F.max_pool2d(<span class="va">self</span>.conv1(x), <span class="dv">2</span>))</span>
+<span id="cb17-67"><a href="#cb17-67" tabindex="-1"></a>        x <span class="op">=</span> F.relu(F.max_pool2d(<span class="va">self</span>.conv2(x), <span class="dv">2</span>))</span>
+<span id="cb17-68"><a href="#cb17-68" tabindex="-1"></a>        x <span class="op">=</span> x.view(<span class="op">-</span><span class="dv">1</span>, <span class="dv">64</span><span class="op">*</span><span class="dv">5</span><span class="op">*</span><span class="dv">5</span>)  <span class="co"># Updated this line</span></span>
+<span id="cb17-69"><a href="#cb17-69" tabindex="-1"></a>        x <span class="op">=</span> F.relu(<span class="va">self</span>.fc1(x))</span>
+<span id="cb17-70"><a href="#cb17-70" tabindex="-1"></a>        x <span class="op">=</span> <span class="va">self</span>.fc2(x)</span>
+<span id="cb17-71"><a href="#cb17-71" tabindex="-1"></a>        <span class="cf">return</span> x</span>
+<span id="cb17-72"><a href="#cb17-72" tabindex="-1"></a></span>
+<span id="cb17-73"><a href="#cb17-73" tabindex="-1"></a>device <span class="op">=</span> torch.device(<span class="st">'cuda'</span> <span class="cf">if</span> torch.cuda.is_available() <span class="cf">else</span> <span class="st">'cpu'</span>)</span>
+<span id="cb17-74"><a href="#cb17-74" tabindex="-1"></a>model <span class="op">=</span> SimpleCNN().to(device)</span>
+<span id="cb17-75"><a href="#cb17-75" tabindex="-1"></a>criterion <span class="op">=</span> nn.CrossEntropyLoss()</span>
+<span id="cb17-76"><a href="#cb17-76" tabindex="-1"></a>optimizer <span class="op">=</span> optim.Adam(model.parameters(), lr<span class="op">=</span><span class="fl">0.001</span>)</span>
+<span id="cb17-77"><a href="#cb17-77" tabindex="-1"></a></span>
+<span id="cb17-78"><a href="#cb17-78" tabindex="-1"></a><span class="kw">def</span> train_model(model, train_loader, criterion, optimizer, epochs<span class="op">=</span><span class="dv">5</span>):</span>
+<span id="cb17-79"><a href="#cb17-79" tabindex="-1"></a>    model.train()</span>
+<span id="cb17-80"><a href="#cb17-80" tabindex="-1"></a>    <span class="cf">for</span> epoch <span class="kw">in</span> <span class="bu">range</span>(epochs):</span>
+<span id="cb17-81"><a href="#cb17-81" tabindex="-1"></a>        running_loss <span class="op">=</span> <span class="fl">0.0</span></span>
+<span id="cb17-82"><a href="#cb17-82" tabindex="-1"></a>        <span class="cf">for</span> inputs, labels <span class="kw">in</span> train_loader:</span>
+<span id="cb17-83"><a href="#cb17-83" tabindex="-1"></a>            inputs, labels <span class="op">=</span> inputs.to(device), labels.to(device)</span>
+<span id="cb17-84"><a href="#cb17-84" tabindex="-1"></a>            optimizer.zero_grad()</span>
+<span id="cb17-85"><a href="#cb17-85" tabindex="-1"></a>            outputs <span class="op">=</span> model(inputs)</span>
+<span id="cb17-86"><a href="#cb17-86" tabindex="-1"></a>            loss <span class="op">=</span> criterion(outputs, labels)</span>
+<span id="cb17-87"><a href="#cb17-87" tabindex="-1"></a>            loss.backward()</span>
+<span id="cb17-88"><a href="#cb17-88" tabindex="-1"></a>            optimizer.step()</span>
+<span id="cb17-89"><a href="#cb17-89" tabindex="-1"></a>            running_loss <span class="op">+=</span> loss.item()</span>
+<span id="cb17-90"><a href="#cb17-90" tabindex="-1"></a>        <span class="bu">print</span>(<span class="ss">f'Epoch </span><span class="sc">{</span>epoch<span class="op">+</span><span class="dv">1</span><span class="sc">}</span><span class="ss">, Loss: </span><span class="sc">{</span>running_loss<span class="op">/</span><span class="bu">len</span>(train_loader)<span class="sc">}</span><span class="ss">'</span>)</span>
+<span id="cb17-91"><a href="#cb17-91" tabindex="-1"></a></span>
+<span id="cb17-92"><a href="#cb17-92" tabindex="-1"></a>train_model(model, train_loader, criterion, optimizer)</span></code></pre>
+</div>
+<div class="codewrapper sourceCode" id="cb18">
+<h3 class="code-label">PYTHON<i aria-hidden="true" data-feather="chevron-left"></i><i aria-hidden="true" data-feather="chevron-right"></i>
+</h3>
+<pre class="sourceCode python" tabindex="0"><code class="sourceCode python"><span id="cb18-1"><a href="#cb18-1" tabindex="-1"></a><span class="im">from</span> sklearn.metrics <span class="im">import</span> confusion_matrix, ConfusionMatrixDisplay</span>
+<span id="cb18-2"><a href="#cb18-2" tabindex="-1"></a></span>
+<span id="cb18-3"><a href="#cb18-3" tabindex="-1"></a><span class="co"># Function to plot confusion matrix</span></span>
+<span id="cb18-4"><a href="#cb18-4" tabindex="-1"></a><span class="kw">def</span> plot_confusion_matrix(labels, predictions, title):</span>
+<span id="cb18-5"><a href="#cb18-5" tabindex="-1"></a>    cm <span class="op">=</span> confusion_matrix(labels, predictions, labels<span class="op">=</span>[<span class="dv">0</span>, <span class="dv">1</span>])</span>
+<span id="cb18-6"><a href="#cb18-6" tabindex="-1"></a>    disp <span class="op">=</span> ConfusionMatrixDisplay(confusion_matrix<span class="op">=</span>cm, display_labels<span class="op">=</span>[<span class="st">"T-shirt/top"</span>, <span class="st">"Trouser"</span>])</span>
+<span id="cb18-7"><a href="#cb18-7" tabindex="-1"></a>    disp.plot(cmap<span class="op">=</span>plt.cm.Blues)</span>
+<span id="cb18-8"><a href="#cb18-8" tabindex="-1"></a>    plt.title(title)</span>
+<span id="cb18-9"><a href="#cb18-9" tabindex="-1"></a>    plt.show()</span>
+<span id="cb18-10"><a href="#cb18-10" tabindex="-1"></a></span>
+<span id="cb18-11"><a href="#cb18-11" tabindex="-1"></a><span class="co"># Function to evaluate model on a dataset</span></span>
+<span id="cb18-12"><a href="#cb18-12" tabindex="-1"></a><span class="kw">def</span> evaluate_model(model, dataloader, device):</span>
+<span id="cb18-13"><a href="#cb18-13" tabindex="-1"></a>    model.<span class="bu">eval</span>()</span>
+<span id="cb18-14"><a href="#cb18-14" tabindex="-1"></a>    all_labels <span class="op">=</span> []</span>
+<span id="cb18-15"><a href="#cb18-15" tabindex="-1"></a>    all_predictions <span class="op">=</span> []</span>
+<span id="cb18-16"><a href="#cb18-16" tabindex="-1"></a>    <span class="cf">with</span> torch.no_grad():</span>
+<span id="cb18-17"><a href="#cb18-17" tabindex="-1"></a>        <span class="cf">for</span> inputs, labels <span class="kw">in</span> dataloader:</span>
+<span id="cb18-18"><a href="#cb18-18" tabindex="-1"></a>            inputs, labels <span class="op">=</span> inputs.to(device), labels.to(device)</span>
+<span id="cb18-19"><a href="#cb18-19" tabindex="-1"></a>            outputs <span class="op">=</span> model(inputs)</span>
+<span id="cb18-20"><a href="#cb18-20" tabindex="-1"></a>            _, preds <span class="op">=</span> torch.<span class="bu">max</span>(outputs, <span class="dv">1</span>)</span>
+<span id="cb18-21"><a href="#cb18-21" tabindex="-1"></a>            all_labels.extend(labels.cpu().numpy())</span>
+<span id="cb18-22"><a href="#cb18-22" tabindex="-1"></a>            all_predictions.extend(preds.cpu().numpy())</span>
+<span id="cb18-23"><a href="#cb18-23" tabindex="-1"></a>    <span class="cf">return</span> np.array(all_labels), np.array(all_predictions)</span>
+<span id="cb18-24"><a href="#cb18-24" tabindex="-1"></a></span>
+<span id="cb18-25"><a href="#cb18-25" tabindex="-1"></a><span class="co"># Evaluate on train data</span></span>
+<span id="cb18-26"><a href="#cb18-26" tabindex="-1"></a>train_labels, train_predictions <span class="op">=</span> evaluate_model(model, train_loader, device)</span>
+<span id="cb18-27"><a href="#cb18-27" tabindex="-1"></a>plot_confusion_matrix(train_labels, train_predictions, <span class="st">"Confusion Matrix for Train Data"</span>)</span>
+<span id="cb18-28"><a href="#cb18-28" tabindex="-1"></a></span>
+<span id="cb18-29"><a href="#cb18-29" tabindex="-1"></a><span class="co"># Evaluate on test data</span></span>
+<span id="cb18-30"><a href="#cb18-30" tabindex="-1"></a>test_labels, test_predictions <span class="op">=</span> evaluate_model(model, test_loader, device)</span>
+<span id="cb18-31"><a href="#cb18-31" tabindex="-1"></a>plot_confusion_matrix(test_labels, test_predictions, <span class="st">"Confusion Matrix for Test Data"</span>)</span></code></pre>
+</div>
+<div class="codewrapper sourceCode" id="cb19">
+<h3 class="code-label">PYTHON<i aria-hidden="true" data-feather="chevron-left"></i><i aria-hidden="true" data-feather="chevron-right"></i>
+</h3>
+<pre class="sourceCode python" tabindex="0"><code class="sourceCode python"><span id="cb19-1"><a href="#cb19-1" tabindex="-1"></a><span class="im">from</span> scipy.stats <span class="im">import</span> gaussian_kde</span>
+<span id="cb19-2"><a href="#cb19-2" tabindex="-1"></a><span class="im">from</span> pytorch_ood.detector <span class="im">import</span> EnergyBased</span>
+<span id="cb19-3"><a href="#cb19-3" tabindex="-1"></a><span class="im">from</span> sklearn.metrics <span class="im">import</span> precision_recall_fscore_support, accuracy_score</span>
+<span id="cb19-4"><a href="#cb19-4" tabindex="-1"></a></span>
+<span id="cb19-5"><a href="#cb19-5" tabindex="-1"></a><span class="co"># Compute softmax scores</span></span>
+<span id="cb19-6"><a href="#cb19-6" tabindex="-1"></a><span class="kw">def</span> get_softmax_scores(model, dataloader):</span>
+<span id="cb19-7"><a href="#cb19-7" tabindex="-1"></a>    model.<span class="bu">eval</span>()</span>
+<span id="cb19-8"><a href="#cb19-8" tabindex="-1"></a>    softmax_scores <span class="op">=</span> []</span>
+<span id="cb19-9"><a href="#cb19-9" tabindex="-1"></a>    <span class="cf">with</span> torch.no_grad():</span>
+<span id="cb19-10"><a href="#cb19-10" tabindex="-1"></a>        <span class="cf">for</span> inputs, _ <span class="kw">in</span> dataloader:</span>
+<span id="cb19-11"><a href="#cb19-11" tabindex="-1"></a>            inputs <span class="op">=</span> inputs.to(device)</span>
+<span id="cb19-12"><a href="#cb19-12" tabindex="-1"></a>            outputs <span class="op">=</span> model(inputs)</span>
+<span id="cb19-13"><a href="#cb19-13" tabindex="-1"></a>            softmax <span class="op">=</span> torch.nn.functional.softmax(outputs, dim<span class="op">=</span><span class="dv">1</span>)</span>
+<span id="cb19-14"><a href="#cb19-14" tabindex="-1"></a>            softmax_scores.extend(softmax.cpu().numpy())</span>
+<span id="cb19-15"><a href="#cb19-15" tabindex="-1"></a>    <span class="cf">return</span> np.array(softmax_scores)</span>
+<span id="cb19-16"><a href="#cb19-16" tabindex="-1"></a></span>
+<span id="cb19-17"><a href="#cb19-17" tabindex="-1"></a>id_softmax_scores <span class="op">=</span> get_softmax_scores(model, test_loader)</span>
+<span id="cb19-18"><a href="#cb19-18" tabindex="-1"></a>ood_softmax_scores <span class="op">=</span> get_softmax_scores(model, ood_loader)</span>
+<span id="cb19-19"><a href="#cb19-19" tabindex="-1"></a></span>
+<span id="cb19-20"><a href="#cb19-20" tabindex="-1"></a><span class="co"># Initialize the energy-based OOD detector</span></span>
+<span id="cb19-21"><a href="#cb19-21" tabindex="-1"></a>energy_detector <span class="op">=</span> EnergyBased(model, t<span class="op">=</span><span class="fl">1.0</span>)</span>
+<span id="cb19-22"><a href="#cb19-22" tabindex="-1"></a></span>
+<span id="cb19-23"><a href="#cb19-23" tabindex="-1"></a><span class="co"># Compute energy scores</span></span>
+<span id="cb19-24"><a href="#cb19-24" tabindex="-1"></a><span class="kw">def</span> get_energy_scores(detector, dataloader):</span>
+<span id="cb19-25"><a href="#cb19-25" tabindex="-1"></a>    scores <span class="op">=</span> []</span>
+<span id="cb19-26"><a href="#cb19-26" tabindex="-1"></a>    detector.model.<span class="bu">eval</span>()</span>
+<span id="cb19-27"><a href="#cb19-27" tabindex="-1"></a>    <span class="cf">with</span> torch.no_grad():</span>
+<span id="cb19-28"><a href="#cb19-28" tabindex="-1"></a>        <span class="cf">for</span> inputs, _ <span class="kw">in</span> dataloader:</span>
+<span id="cb19-29"><a href="#cb19-29" tabindex="-1"></a>            inputs <span class="op">=</span> inputs.to(device)</span>
+<span id="cb19-30"><a href="#cb19-30" tabindex="-1"></a>            score <span class="op">=</span> detector.predict(inputs)</span>
+<span id="cb19-31"><a href="#cb19-31" tabindex="-1"></a>            scores.extend(score.cpu().numpy())</span>
+<span id="cb19-32"><a href="#cb19-32" tabindex="-1"></a>    <span class="cf">return</span> np.array(scores)</span>
+<span id="cb19-33"><a href="#cb19-33" tabindex="-1"></a></span>
+<span id="cb19-34"><a href="#cb19-34" tabindex="-1"></a>id_energy_scores <span class="op">=</span> get_energy_scores(energy_detector, test_loader)</span>
+<span id="cb19-35"><a href="#cb19-35" tabindex="-1"></a>ood_energy_scores <span class="op">=</span> get_energy_scores(energy_detector, ood_loader)</span>
+<span id="cb19-36"><a href="#cb19-36" tabindex="-1"></a></span>
+<span id="cb19-37"><a href="#cb19-37" tabindex="-1"></a><span class="im">import</span> matplotlib.pyplot <span class="im">as</span> plt</span>
+<span id="cb19-38"><a href="#cb19-38" tabindex="-1"></a></span>
+<span id="cb19-39"><a href="#cb19-39" tabindex="-1"></a></span>
+<span id="cb19-40"><a href="#cb19-40" tabindex="-1"></a><span class="co"># Plot PSDs</span></span>
+<span id="cb19-41"><a href="#cb19-41" tabindex="-1"></a></span>
+<span id="cb19-42"><a href="#cb19-42" tabindex="-1"></a><span class="co"># Function to plot PSD</span></span>
+<span id="cb19-43"><a href="#cb19-43" tabindex="-1"></a><span class="kw">def</span> plot_psd(id_scores, ood_scores, method_name):</span>
+<span id="cb19-44"><a href="#cb19-44" tabindex="-1"></a>    plt.figure(figsize<span class="op">=</span>(<span class="dv">12</span>, <span class="dv">6</span>))</span>
+<span id="cb19-45"><a href="#cb19-45" tabindex="-1"></a>    alpha <span class="op">=</span> <span class="fl">0.3</span></span>
+<span id="cb19-46"><a href="#cb19-46" tabindex="-1"></a></span>
+<span id="cb19-47"><a href="#cb19-47" tabindex="-1"></a>    <span class="co"># Plot PSD for ID scores</span></span>
+<span id="cb19-48"><a href="#cb19-48" tabindex="-1"></a>    id_density <span class="op">=</span> gaussian_kde(id_scores)</span>
+<span id="cb19-49"><a href="#cb19-49" tabindex="-1"></a>    x_id <span class="op">=</span> np.linspace(id_scores.<span class="bu">min</span>(), id_scores.<span class="bu">max</span>(), <span class="dv">1000</span>)</span>
+<span id="cb19-50"><a href="#cb19-50" tabindex="-1"></a>    plt.plot(x_id, id_density(x_id), label<span class="op">=</span><span class="ss">f'ID (</span><span class="sc">{</span>method_name<span class="sc">}</span><span class="ss">)'</span>, color<span class="op">=</span><span class="st">'blue'</span>, alpha<span class="op">=</span>alpha)</span>
+<span id="cb19-51"><a href="#cb19-51" tabindex="-1"></a></span>
+<span id="cb19-52"><a href="#cb19-52" tabindex="-1"></a>    <span class="co"># Plot PSD for OOD scores</span></span>
+<span id="cb19-53"><a href="#cb19-53" tabindex="-1"></a>    ood_density <span class="op">=</span> gaussian_kde(ood_scores)</span>
+<span id="cb19-54"><a href="#cb19-54" tabindex="-1"></a>    x_ood <span class="op">=</span> np.linspace(ood_scores.<span class="bu">min</span>(), ood_scores.<span class="bu">max</span>(), <span class="dv">1000</span>)</span>
+<span id="cb19-55"><a href="#cb19-55" tabindex="-1"></a>    plt.plot(x_ood, ood_density(x_ood), label<span class="op">=</span><span class="ss">f'OOD (</span><span class="sc">{</span>method_name<span class="sc">}</span><span class="ss">)'</span>, color<span class="op">=</span><span class="st">'red'</span>, alpha<span class="op">=</span>alpha)</span>
+<span id="cb19-56"><a href="#cb19-56" tabindex="-1"></a></span>
+<span id="cb19-57"><a href="#cb19-57" tabindex="-1"></a>    plt.xlabel(<span class="st">'Score'</span>)</span>
+<span id="cb19-58"><a href="#cb19-58" tabindex="-1"></a>    plt.ylabel(<span class="st">'Density'</span>)</span>
+<span id="cb19-59"><a href="#cb19-59" tabindex="-1"></a>    plt.title(<span class="ss">f'Probability Density Distributions for </span><span class="sc">{</span>method_name<span class="sc">}</span><span class="ss"> Scores'</span>)</span>
+<span id="cb19-60"><a href="#cb19-60" tabindex="-1"></a>    plt.legend()</span>
+<span id="cb19-61"><a href="#cb19-61" tabindex="-1"></a>    plt.show()</span>
+<span id="cb19-62"><a href="#cb19-62" tabindex="-1"></a></span>
+<span id="cb19-63"><a href="#cb19-63" tabindex="-1"></a><span class="co"># Plot PSD for softmax scores</span></span>
+<span id="cb19-64"><a href="#cb19-64" tabindex="-1"></a>plot_psd(id_softmax_scores[:, <span class="dv">1</span>], ood_softmax_scores[:, <span class="dv">1</span>], <span class="st">'Softmax'</span>)</span>
+<span id="cb19-65"><a href="#cb19-65" tabindex="-1"></a></span>
+<span id="cb19-66"><a href="#cb19-66" tabindex="-1"></a><span class="co"># Plot PSD for energy scores</span></span>
+<span id="cb19-67"><a href="#cb19-67" tabindex="-1"></a>plot_psd(id_energy_scores, ood_energy_scores, <span class="st">'Energy'</span>)</span>
+<span id="cb19-68"><a href="#cb19-68" tabindex="-1"></a></span></code></pre>
+</div>
+<div class="codewrapper sourceCode" id="cb20">
+<h3 class="code-label">PYTHON<i aria-hidden="true" data-feather="chevron-left"></i><i aria-hidden="true" data-feather="chevron-right"></i>
+</h3>
+<pre class="sourceCode python" tabindex="0"><code class="sourceCode python"><span id="cb20-1"><a href="#cb20-1" tabindex="-1"></a><span class="im">import</span> numpy <span class="im">as</span> np</span>
+<span id="cb20-2"><a href="#cb20-2" tabindex="-1"></a><span class="im">import</span> matplotlib.pyplot <span class="im">as</span> plt</span>
+<span id="cb20-3"><a href="#cb20-3" tabindex="-1"></a><span class="im">from</span> sklearn.metrics <span class="im">import</span> precision_recall_fscore_support, accuracy_score, confusion_matrix, ConfusionMatrixDisplay</span>
+<span id="cb20-4"><a href="#cb20-4" tabindex="-1"></a></span>
+<span id="cb20-5"><a href="#cb20-5" tabindex="-1"></a><span class="co"># Define thresholds to evaluate</span></span>
+<span id="cb20-6"><a href="#cb20-6" tabindex="-1"></a>thresholds <span class="op">=</span> np.linspace(id_energy_scores.<span class="bu">min</span>(), id_energy_scores.<span class="bu">max</span>(), <span class="dv">50</span>)</span>
+<span id="cb20-7"><a href="#cb20-7" tabindex="-1"></a></span>
+<span id="cb20-8"><a href="#cb20-8" tabindex="-1"></a><span class="co"># Store evaluation metrics for each threshold</span></span>
+<span id="cb20-9"><a href="#cb20-9" tabindex="-1"></a>accuracies <span class="op">=</span> []</span>
+<span id="cb20-10"><a href="#cb20-10" tabindex="-1"></a>precisions <span class="op">=</span> []</span>
+<span id="cb20-11"><a href="#cb20-11" tabindex="-1"></a>recalls <span class="op">=</span> []</span>
+<span id="cb20-12"><a href="#cb20-12" tabindex="-1"></a>f1_scores <span class="op">=</span> []</span>
+<span id="cb20-13"><a href="#cb20-13" tabindex="-1"></a></span>
+<span id="cb20-14"><a href="#cb20-14" tabindex="-1"></a><span class="co"># True labels for OOD data (since they are not part of the original labels)</span></span>
+<span id="cb20-15"><a href="#cb20-15" tabindex="-1"></a>ood_true_labels <span class="op">=</span> np.full(<span class="bu">len</span>(ood_energy_scores), <span class="op">-</span><span class="dv">1</span>)</span>
+<span id="cb20-16"><a href="#cb20-16" tabindex="-1"></a></span>
+<span id="cb20-17"><a href="#cb20-17" tabindex="-1"></a><span class="co"># We need the test_labels to be aligned with the ID data</span></span>
+<span id="cb20-18"><a href="#cb20-18" tabindex="-1"></a>id_true_labels <span class="op">=</span> test_labels[:<span class="bu">len</span>(id_energy_scores)]</span>
+<span id="cb20-19"><a href="#cb20-19" tabindex="-1"></a></span>
+<span id="cb20-20"><a href="#cb20-20" tabindex="-1"></a><span class="cf">for</span> threshold <span class="kw">in</span> thresholds:</span>
+<span id="cb20-21"><a href="#cb20-21" tabindex="-1"></a>    <span class="co"># Classify OOD examples based on energy scores</span></span>
+<span id="cb20-22"><a href="#cb20-22" tabindex="-1"></a>    ood_classifications <span class="op">=</span> np.where(ood_energy_scores <span class="op">&gt;=</span> threshold, <span class="op">-</span><span class="dv">1</span>,  <span class="co"># classified as OOD</span></span>
+<span id="cb20-23"><a href="#cb20-23" tabindex="-1"></a>                                   np.where(ood_energy_scores <span class="op">&lt;</span> threshold, <span class="dv">0</span>, <span class="op">-</span><span class="dv">1</span>))  <span class="co"># classified as ID</span></span>
+<span id="cb20-24"><a href="#cb20-24" tabindex="-1"></a></span>
+<span id="cb20-25"><a href="#cb20-25" tabindex="-1"></a>    <span class="co"># Classify ID examples based on energy scores</span></span>
+<span id="cb20-26"><a href="#cb20-26" tabindex="-1"></a>    id_classifications <span class="op">=</span> np.where(id_energy_scores <span class="op">&gt;=</span> threshold, <span class="op">-</span><span class="dv">1</span>,  <span class="co"># classified as OOD</span></span>
+<span id="cb20-27"><a href="#cb20-27" tabindex="-1"></a>                                  np.where(id_energy_scores <span class="op">&lt;</span> threshold, id_true_labels, <span class="op">-</span><span class="dv">1</span>))  <span class="co"># classified as ID</span></span>
+<span id="cb20-28"><a href="#cb20-28" tabindex="-1"></a></span>
+<span id="cb20-29"><a href="#cb20-29" tabindex="-1"></a>    <span class="co"># Combine OOD and ID classifications and true labels</span></span>
+<span id="cb20-30"><a href="#cb20-30" tabindex="-1"></a>    all_predictions <span class="op">=</span> np.concatenate([ood_classifications, id_classifications])</span>
+<span id="cb20-31"><a href="#cb20-31" tabindex="-1"></a>    all_true_labels <span class="op">=</span> np.concatenate([ood_true_labels, id_true_labels])</span>
+<span id="cb20-32"><a href="#cb20-32" tabindex="-1"></a></span>
+<span id="cb20-33"><a href="#cb20-33" tabindex="-1"></a>    <span class="co"># Evaluate metrics</span></span>
+<span id="cb20-34"><a href="#cb20-34" tabindex="-1"></a>    precision, recall, f1, _ <span class="op">=</span> precision_recall_fscore_support(all_true_labels, all_predictions, labels<span class="op">=</span>[<span class="dv">0</span>, <span class="dv">1</span>], average<span class="op">=</span><span class="st">'macro'</span>, zero_division<span class="op">=</span><span class="dv">0</span>)</span>
+<span id="cb20-35"><a href="#cb20-35" tabindex="-1"></a>    accuracy <span class="op">=</span> accuracy_score(all_true_labels, all_predictions)</span>
+<span id="cb20-36"><a href="#cb20-36" tabindex="-1"></a></span>
+<span id="cb20-37"><a href="#cb20-37" tabindex="-1"></a>    accuracies.append(accuracy)</span>
+<span id="cb20-38"><a href="#cb20-38" tabindex="-1"></a>    precisions.append(precision)</span>
+<span id="cb20-39"><a href="#cb20-39" tabindex="-1"></a>    recalls.append(recall)</span>
+<span id="cb20-40"><a href="#cb20-40" tabindex="-1"></a>    f1_scores.append(f1)</span>
+<span id="cb20-41"><a href="#cb20-41" tabindex="-1"></a></span>
+<span id="cb20-42"><a href="#cb20-42" tabindex="-1"></a><span class="co"># Find the best thresholds for each metric</span></span>
+<span id="cb20-43"><a href="#cb20-43" tabindex="-1"></a>best_f1_index <span class="op">=</span> np.argmax(f1_scores)</span>
+<span id="cb20-44"><a href="#cb20-44" tabindex="-1"></a>best_f1_threshold <span class="op">=</span> thresholds[best_f1_index]</span>
+<span id="cb20-45"><a href="#cb20-45" tabindex="-1"></a></span>
+<span id="cb20-46"><a href="#cb20-46" tabindex="-1"></a>best_precision_index <span class="op">=</span> np.argmax(precisions)</span>
+<span id="cb20-47"><a href="#cb20-47" tabindex="-1"></a>best_precision_threshold <span class="op">=</span> thresholds[best_precision_index]</span>
+<span id="cb20-48"><a href="#cb20-48" tabindex="-1"></a></span>
+<span id="cb20-49"><a href="#cb20-49" tabindex="-1"></a>best_recall_index <span class="op">=</span> np.argmax(recalls)</span>
+<span id="cb20-50"><a href="#cb20-50" tabindex="-1"></a>best_recall_threshold <span class="op">=</span> thresholds[best_recall_index]</span>
+<span id="cb20-51"><a href="#cb20-51" tabindex="-1"></a></span>
+<span id="cb20-52"><a href="#cb20-52" tabindex="-1"></a><span class="bu">print</span>(<span class="ss">f"Best F1 threshold: </span><span class="sc">{</span>best_f1_threshold<span class="sc">}</span><span class="ss">, F1 Score: </span><span class="sc">{</span>f1_scores[best_f1_index]<span class="sc">}</span><span class="ss">"</span>)</span>
+<span id="cb20-53"><a href="#cb20-53" tabindex="-1"></a><span class="bu">print</span>(<span class="ss">f"Best Precision threshold: </span><span class="sc">{</span>best_precision_threshold<span class="sc">}</span><span class="ss">, Precision: </span><span class="sc">{</span>precisions[best_precision_index]<span class="sc">}</span><span class="ss">"</span>)</span>
+<span id="cb20-54"><a href="#cb20-54" tabindex="-1"></a><span class="bu">print</span>(<span class="ss">f"Best Recall threshold: </span><span class="sc">{</span>best_recall_threshold<span class="sc">}</span><span class="ss">, Recall: </span><span class="sc">{</span>recalls[best_recall_index]<span class="sc">}</span><span class="ss">"</span>)</span>
+<span id="cb20-55"><a href="#cb20-55" tabindex="-1"></a></span>
+<span id="cb20-56"><a href="#cb20-56" tabindex="-1"></a><span class="co"># Plot metrics as functions of the threshold</span></span>
+<span id="cb20-57"><a href="#cb20-57" tabindex="-1"></a>plt.figure(figsize<span class="op">=</span>(<span class="dv">12</span>, <span class="dv">8</span>))</span>
+<span id="cb20-58"><a href="#cb20-58" tabindex="-1"></a>plt.plot(thresholds, precisions, label<span class="op">=</span><span class="st">'Precision'</span>, color<span class="op">=</span><span class="st">'g'</span>)</span>
+<span id="cb20-59"><a href="#cb20-59" tabindex="-1"></a>plt.plot(thresholds, recalls, label<span class="op">=</span><span class="st">'Recall'</span>, color<span class="op">=</span><span class="st">'b'</span>)</span>
+<span id="cb20-60"><a href="#cb20-60" tabindex="-1"></a>plt.plot(thresholds, f1_scores, label<span class="op">=</span><span class="st">'F1 Score'</span>, color<span class="op">=</span><span class="st">'r'</span>)</span>
+<span id="cb20-61"><a href="#cb20-61" tabindex="-1"></a></span>
+<span id="cb20-62"><a href="#cb20-62" tabindex="-1"></a><span class="co"># Add best threshold indicators</span></span>
+<span id="cb20-63"><a href="#cb20-63" tabindex="-1"></a>plt.axvline(x<span class="op">=</span>best_f1_threshold, color<span class="op">=</span><span class="st">'r'</span>, linestyle<span class="op">=</span><span class="st">'--'</span>, label<span class="op">=</span><span class="ss">f'Best F1 Threshold: </span><span class="sc">{</span>best_f1_threshold<span class="sc">:.2f}</span><span class="ss">'</span>)</span>
+<span id="cb20-64"><a href="#cb20-64" tabindex="-1"></a>plt.axvline(x<span class="op">=</span>best_precision_threshold, color<span class="op">=</span><span class="st">'g'</span>, linestyle<span class="op">=</span><span class="st">'--'</span>, label<span class="op">=</span><span class="ss">f'Best Precision Threshold: </span><span class="sc">{</span>best_precision_threshold<span class="sc">:.2f}</span><span class="ss">'</span>)</span>
+<span id="cb20-65"><a href="#cb20-65" tabindex="-1"></a>plt.axvline(x<span class="op">=</span>best_recall_threshold, color<span class="op">=</span><span class="st">'b'</span>, linestyle<span class="op">=</span><span class="st">'--'</span>, label<span class="op">=</span><span class="ss">f'Best Recall Threshold: </span><span class="sc">{</span>best_recall_threshold<span class="sc">:.2f}</span><span class="ss">'</span>)</span>
+<span id="cb20-66"><a href="#cb20-66" tabindex="-1"></a></span>
+<span id="cb20-67"><a href="#cb20-67" tabindex="-1"></a>plt.xlabel(<span class="st">'Threshold'</span>)</span>
+<span id="cb20-68"><a href="#cb20-68" tabindex="-1"></a>plt.ylabel(<span class="st">'Metric Value'</span>)</span>
+<span id="cb20-69"><a href="#cb20-69" tabindex="-1"></a>plt.title(<span class="st">'Evaluation Metrics as Functions of Threshold (Energy-Based OOD Detection)'</span>)</span>
+<span id="cb20-70"><a href="#cb20-70" tabindex="-1"></a>plt.legend()</span>
+<span id="cb20-71"><a href="#cb20-71" tabindex="-1"></a>plt.show()</span></code></pre>
+</div>
+<div class="codewrapper sourceCode" id="cb21">
+<h3 class="code-label">PYTHON<i aria-hidden="true" data-feather="chevron-left"></i><i aria-hidden="true" data-feather="chevron-right"></i>
+</h3>
+<pre class="sourceCode python" tabindex="0"><code class="sourceCode python"><span id="cb21-1"><a href="#cb21-1" tabindex="-1"></a><span class="im">import</span> numpy <span class="im">as</span> np</span>
+<span id="cb21-2"><a href="#cb21-2" tabindex="-1"></a><span class="im">import</span> matplotlib.pyplot <span class="im">as</span> plt</span>
+<span id="cb21-3"><a href="#cb21-3" tabindex="-1"></a><span class="im">from</span> sklearn.metrics <span class="im">import</span> confusion_matrix, ConfusionMatrixDisplay</span>
+<span id="cb21-4"><a href="#cb21-4" tabindex="-1"></a></span>
+<span id="cb21-5"><a href="#cb21-5" tabindex="-1"></a><span class="co"># Threshold value for the energy score</span></span>
+<span id="cb21-6"><a href="#cb21-6" tabindex="-1"></a>upper_threshold <span class="op">=</span> best_f1_threshold  <span class="co"># Using the best F1 threshold from the previous calculation</span></span>
+<span id="cb21-7"><a href="#cb21-7" tabindex="-1"></a></span>
+<span id="cb21-8"><a href="#cb21-8" tabindex="-1"></a><span class="co"># Classifying OOD examples based on energy scores</span></span>
+<span id="cb21-9"><a href="#cb21-9" tabindex="-1"></a>ood_classifications <span class="op">=</span> np.where(ood_energy_scores <span class="op">&gt;=</span> upper_threshold, <span class="op">-</span><span class="dv">1</span>,  <span class="co"># classified as OOD</span></span>
+<span id="cb21-10"><a href="#cb21-10" tabindex="-1"></a>                               np.where(ood_energy_scores <span class="op">&lt;</span> upper_threshold, <span class="dv">0</span>, <span class="op">-</span><span class="dv">1</span>))  <span class="co"># classified as ID</span></span>
+<span id="cb21-11"><a href="#cb21-11" tabindex="-1"></a></span>
+<span id="cb21-12"><a href="#cb21-12" tabindex="-1"></a><span class="co"># Classifying ID examples based on energy scores</span></span>
+<span id="cb21-13"><a href="#cb21-13" tabindex="-1"></a>id_classifications <span class="op">=</span> np.where(id_energy_scores <span class="op">&gt;=</span> upper_threshold, <span class="op">-</span><span class="dv">1</span>,  <span class="co"># classified as OOD</span></span>
+<span id="cb21-14"><a href="#cb21-14" tabindex="-1"></a>                              np.where(id_energy_scores <span class="op">&lt;</span> upper_threshold, id_true_labels, <span class="op">-</span><span class="dv">1</span>))  <span class="co"># classified as ID</span></span>
+<span id="cb21-15"><a href="#cb21-15" tabindex="-1"></a></span>
+<span id="cb21-16"><a href="#cb21-16" tabindex="-1"></a><span class="co"># Combine OOD and ID classifications and true labels</span></span>
+<span id="cb21-17"><a href="#cb21-17" tabindex="-1"></a>all_predictions <span class="op">=</span> np.concatenate([ood_classifications, id_classifications])</span>
+<span id="cb21-18"><a href="#cb21-18" tabindex="-1"></a>all_true_labels <span class="op">=</span> np.concatenate([ood_true_labels, id_true_labels])</span>
+<span id="cb21-19"><a href="#cb21-19" tabindex="-1"></a></span>
+<span id="cb21-20"><a href="#cb21-20" tabindex="-1"></a><span class="co"># Confusion matrix</span></span>
+<span id="cb21-21"><a href="#cb21-21" tabindex="-1"></a>cm <span class="op">=</span> confusion_matrix(all_true_labels, all_predictions, labels<span class="op">=</span>[<span class="dv">0</span>, <span class="dv">1</span>, <span class="op">-</span><span class="dv">1</span>])</span>
+<span id="cb21-22"><a href="#cb21-22" tabindex="-1"></a></span>
+<span id="cb21-23"><a href="#cb21-23" tabindex="-1"></a><span class="co"># Plotting the confusion matrix</span></span>
+<span id="cb21-24"><a href="#cb21-24" tabindex="-1"></a>disp <span class="op">=</span> ConfusionMatrixDisplay(confusion_matrix<span class="op">=</span>cm, display_labels<span class="op">=</span>[<span class="st">"Shirt"</span>, <span class="st">"Pants"</span>, <span class="st">"OOD"</span>])</span>
+<span id="cb21-25"><a href="#cb21-25" tabindex="-1"></a>disp.plot(cmap<span class="op">=</span>plt.cm.Blues)</span>
+<span id="cb21-26"><a href="#cb21-26" tabindex="-1"></a>plt.title(<span class="st">'Confusion Matrix for OOD and ID Classification (Energy-Based)'</span>)</span>
+<span id="cb21-27"><a href="#cb21-27" tabindex="-1"></a>plt.show()</span></code></pre>
+</div>
+</div>
+</div>
+</div>
+<div class="section level1">
+<h1 id="conclusion">Conclusion<a class="anchor" aria-label="anchor" href="#conclusion"></a></h1>
+<div class="codewrapper sourceCode" id="cb22">
 <h3 class="code-label">PYTHON<i aria-hidden="true" data-feather="chevron-left"></i><i aria-hidden="true" data-feather="chevron-right"></i>
 </h3>
 <pre class="sourceCode python" tabindex="0"><code class="sourceCode python"></code></pre>
 </div>
-<!--
+<div class="section level2">
+<h2 id="references-and-supplemental-resources">References and supplemental resources<a class="anchor" aria-label="anchor" href="#references-and-supplemental-resources"></a></h2>
+<ul><li><a href="https://www.youtube.com/watch?v=hgLC9_9ZCJI" class="external-link uri">https://www.youtube.com/watch?v=hgLC9_9ZCJI</a></li>
+<li>Generalized Out-of-Distribution Detection: A Survey: <a href="https://arxiv.org/abs/2110.11334" class="external-link uri">https://arxiv.org/abs/2110.11334</a> # Glossary</li>
+<li>ID/OOD: In-distribution, out-of-distribution. Generally, the OOD
+instances can be defined as instances (x, y) sampled from an underlying
+distribution other than the training distribution P(Xtrain, Ytrain),
+where Xtrain and Ytrain are the training corpus and training label set,
+respectively.</li>
+<li>OOD instances with semantic shift: OOD instances with semantic shift
+refer to instances that do not belong to y_train. More specifically,
+instances with semantic shift may come from unknown categories or
+irrelevant tasks.</li>
+<li>OOD instances with covariate shift: OOD instances with non-semantic
+shift refer to the instances that belong to y_train but are sampled from
+a distribution other than x_train, e.g., a different
+domain/corpus/location.</li>
+<li>Closed-world assumption: an assumption that the training and test
+data are sampled from the same distribution. However, training data can
+rarely capture the entire distribution. In real-world scenarios,
+out-of-distribution (OOD) instances, which come from categories that are
+not known to the model, can often be present in inference phases.</li>
+<li>Inference-time OOD: After training, use some kind of scoring
+function to determine if test inputs are OOD or not.</li>
+<li>Output-based OOD: Output-based OOD detection methods leverage the
+model’s output distribution to identify OOD instances. These methods
+typically involve analyzing the softmax scores, confidence scores, or
+other output statistics to detect anomalies.</li>
+</ul><!--
 Place links that you need to refer to multiple times across pages here. Delete
 any links that you are not going to use.
- -->
+ --></div>
 </div>
 
 
diff --git a/aio.html b/aio.html
index d76ff2a5..cd39f74b 100644
--- a/aio.html
+++ b/aio.html
@@ -576,7 +576,7 @@ <h3 class="callout-title">Challenge<a class="anchor" aria-label="anchor" href="#
 <button class="accordion-button solution-button collapsed" type="button" data-bs-toggle="collapse" data-bs-target="#collapseSolution1" aria-expanded="false" aria-controls="collapseSolution1">
   <h4 class="accordion-header" id="headingSolution1"> Show me the solution </h4>
 </button>
-<div id="collapseSolution1" class="accordion-collapse collapse" aria-labelledby="headingSolution1" data-bs-parent="#accordionSolution1">
+<div id="collapseSolution1" class="accordion-collapse collapse" data-bs-parent="#accordionSolution1" aria-labelledby="headingSolution1">
 <div class="accordion-body">
 <p>A summary of the principles is listed below:</p>
 <ul>
@@ -3085,7 +3085,7 @@ <h3 class="callout-title">Classifying explanation techniques<a class="anchor" ar
 <button class="accordion-button solution-button collapsed" type="button" data-bs-toggle="collapse" data-bs-target="#collapseSolution1" aria-expanded="false" aria-controls="collapseSolution1">
   <h4 class="accordion-header" id="headingSolution1"> Show me the solution </h4>
 </button>
-<div id="collapseSolution1" class="accordion-collapse collapse" data-bs-parent="#accordionSolution1" aria-labelledby="headingSolution1">
+<div id="collapseSolution1" class="accordion-collapse collapse" aria-labelledby="headingSolution1" data-bs-parent="#accordionSolution1">
 <div class="accordion-body">
 <table class="table">
 <colgroup>
@@ -4264,8 +4264,8 @@ <h3 class="card-title">Questions</h3>
 <ul>
 <li>What are out-of-distribution (OOD) data and why is detecting them
 important in machine learning models?</li>
-<li>How do output-based methods like softmax, energy-based, and
-distance-based methods work for OOD detection?</li>
+<li>How do output-based methods like softmax and energy-based methods
+work for OOD detection?</li>
 <li>What are the limitations of output-based OOD detection methods?</li>
 </ul>
 </div>
@@ -4874,7 +4874,7 @@ <h3 class="code-label">PYTHON<i aria-hidden="true" data-feather="chevron-left"><
 <span id="cb14-29"><a href="#cb14-29" tabindex="-1"></a>  all_true_labels <span class="op">=</span> np.concatenate([<span class="op">-</span><span class="dv">1</span> <span class="op">*</span> np.ones(ood_classifications.shape), train_labels])</span>
 <span id="cb14-30"><a href="#cb14-30" tabindex="-1"></a></span>
 <span id="cb14-31"><a href="#cb14-31" tabindex="-1"></a>  <span class="co"># Evaluate metrics</span></span>
-<span id="cb14-32"><a href="#cb14-32" tabindex="-1"></a>  precision, recall, f1, _ <span class="op">=</span> precision_recall_fscore_support(all_true_labels, all_predictions, labels<span class="op">=</span>[<span class="dv">0</span>, <span class="dv">1</span>], average<span class="op">=</span><span class="st">'macro'</span>) <span class="co"># discuss macro vs micro .</span></span>
+<span id="cb14-32"><a href="#cb14-32" tabindex="-1"></a>  precision, recall, f1, _ <span class="op">=</span> precision_recall_fscore_support(all_true_labels, all_predictions, labels<span class="op">=</span>[<span class="dv">0</span>, <span class="dv">1</span>, <span class="op">-</span><span class="dv">1</span>], average<span class="op">=</span><span class="st">'macro'</span>) <span class="co"># discuss macro vs micro .</span></span>
 <span id="cb14-33"><a href="#cb14-33" tabindex="-1"></a>  accuracy <span class="op">=</span> accuracy_score(all_true_labels, all_predictions)</span>
 <span id="cb14-34"><a href="#cb14-34" tabindex="-1"></a></span>
 <span id="cb14-35"><a href="#cb14-35" tabindex="-1"></a>  accuracies.append(accuracy)</span>
@@ -4913,14 +4913,52 @@ <h3 class="code-label">PYTHON<i aria-hidden="true" data-feather="chevron-left"><
 <span id="cb14-68"><a href="#cb14-68" tabindex="-1"></a>plt.legend()</span>
 <span id="cb14-69"><a href="#cb14-69" tabindex="-1"></a>plt.show()</span></code></pre>
 </div>
+<div class="codewrapper sourceCode" id="cb15">
+<h3 class="code-label">PYTHON<i aria-hidden="true" data-feather="chevron-left"></i><i aria-hidden="true" data-feather="chevron-right"></i>
+</h3>
+<pre class="sourceCode python" tabindex="0"><code class="sourceCode python"><span id="cb15-1"><a href="#cb15-1" tabindex="-1"></a><span class="im">import</span> numpy <span class="im">as</span> np</span>
+<span id="cb15-2"><a href="#cb15-2" tabindex="-1"></a><span class="im">import</span> matplotlib.pyplot <span class="im">as</span> plt</span>
+<span id="cb15-3"><a href="#cb15-3" tabindex="-1"></a><span class="im">from</span> sklearn.metrics <span class="im">import</span> confusion_matrix, ConfusionMatrixDisplay</span>
+<span id="cb15-4"><a href="#cb15-4" tabindex="-1"></a></span>
+<span id="cb15-5"><a href="#cb15-5" tabindex="-1"></a><span class="co"># Assuming ood_probs, id_probs, and train_labels are defined</span></span>
+<span id="cb15-6"><a href="#cb15-6" tabindex="-1"></a><span class="co"># Threshold values</span></span>
+<span id="cb15-7"><a href="#cb15-7" tabindex="-1"></a>upper_threshold <span class="op">=</span> best_f1_threshold</span>
+<span id="cb15-8"><a href="#cb15-8" tabindex="-1"></a></span>
+<span id="cb15-9"><a href="#cb15-9" tabindex="-1"></a><span class="co"># Classifying OOD examples (sandals)</span></span>
+<span id="cb15-10"><a href="#cb15-10" tabindex="-1"></a>ood_classifications <span class="op">=</span> np.where(ood_probs[:, <span class="dv">1</span>] <span class="op">&gt;=</span> upper_threshold, <span class="dv">1</span>,  <span class="co"># classified as pants</span></span>
+<span id="cb15-11"><a href="#cb15-11" tabindex="-1"></a>                               np.where(ood_probs[:, <span class="dv">0</span>] <span class="op">&gt;=</span> upper_threshold, <span class="dv">0</span>,  <span class="co"># classified as shirts</span></span>
+<span id="cb15-12"><a href="#cb15-12" tabindex="-1"></a>                                        <span class="op">-</span><span class="dv">1</span>))  <span class="co"># classified as OOD</span></span>
+<span id="cb15-13"><a href="#cb15-13" tabindex="-1"></a>ood_classifications</span>
+<span id="cb15-14"><a href="#cb15-14" tabindex="-1"></a></span>
+<span id="cb15-15"><a href="#cb15-15" tabindex="-1"></a>id_probs</span>
+<span id="cb15-16"><a href="#cb15-16" tabindex="-1"></a><span class="co"># Classifying ID examples (T-shirts and pants)</span></span>
+<span id="cb15-17"><a href="#cb15-17" tabindex="-1"></a>id_classifications <span class="op">=</span> np.where(id_probs[:, <span class="dv">1</span>] <span class="op">&gt;=</span> upper_threshold, <span class="dv">1</span>,  <span class="co"># classified as pants</span></span>
+<span id="cb15-18"><a href="#cb15-18" tabindex="-1"></a>                              np.where(id_probs[:, <span class="dv">0</span>] <span class="op">&gt;=</span> upper_threshold, <span class="dv">0</span>,  <span class="co"># classified as shirts</span></span>
+<span id="cb15-19"><a href="#cb15-19" tabindex="-1"></a>                                       <span class="op">-</span><span class="dv">1</span>))  <span class="co"># classified as OOD</span></span>
+<span id="cb15-20"><a href="#cb15-20" tabindex="-1"></a></span>
+<span id="cb15-21"><a href="#cb15-21" tabindex="-1"></a>id_classifications</span>
+<span id="cb15-22"><a href="#cb15-22" tabindex="-1"></a></span>
+<span id="cb15-23"><a href="#cb15-23" tabindex="-1"></a><span class="co"># Combine OOD and ID classifications and true labels</span></span>
+<span id="cb15-24"><a href="#cb15-24" tabindex="-1"></a>all_predictions <span class="op">=</span> np.concatenate([ood_classifications, id_classifications])</span>
+<span id="cb15-25"><a href="#cb15-25" tabindex="-1"></a>all_true_labels <span class="op">=</span> np.concatenate([<span class="op">-</span><span class="dv">1</span> <span class="op">*</span> np.ones(ood_classifications.shape), train_labels])</span>
+<span id="cb15-26"><a href="#cb15-26" tabindex="-1"></a></span>
+<span id="cb15-27"><a href="#cb15-27" tabindex="-1"></a><span class="co"># Confusion matrix</span></span>
+<span id="cb15-28"><a href="#cb15-28" tabindex="-1"></a>cm <span class="op">=</span> confusion_matrix(all_true_labels, all_predictions, labels<span class="op">=</span>[<span class="dv">0</span>, <span class="dv">1</span>, <span class="op">-</span><span class="dv">1</span>])</span>
+<span id="cb15-29"><a href="#cb15-29" tabindex="-1"></a></span>
+<span id="cb15-30"><a href="#cb15-30" tabindex="-1"></a><span class="co"># Plotting the confusion matrix</span></span>
+<span id="cb15-31"><a href="#cb15-31" tabindex="-1"></a>disp <span class="op">=</span> ConfusionMatrixDisplay(confusion_matrix<span class="op">=</span>cm, display_labels<span class="op">=</span>[<span class="st">"Shirt"</span>, <span class="st">"Pants"</span>, <span class="st">"OOD"</span>])</span>
+<span id="cb15-32"><a href="#cb15-32" tabindex="-1"></a>disp.plot(cmap<span class="op">=</span>plt.cm.Blues)</span>
+<span id="cb15-33"><a href="#cb15-33" tabindex="-1"></a>plt.title(<span class="st">'Confusion Matrix for OOD and ID Classification'</span>)</span>
+<span id="cb15-34"><a href="#cb15-34" tabindex="-1"></a>plt.show()</span></code></pre>
+</div>
 </div>
 </div>
 </div>
 <div class="section level1">
 <h1 id="example-2-energy-based-ood-detection">Example 2: Energy-Based OOD Detection<a class="anchor" aria-label="anchor" href="#example-2-energy-based-ood-detection"></a>
 </h1>
-<p>Liu et al., Energy-based Out-of-distribution Detection, NeurIPS
-2020</p>
+<p>Liu et al., Energy-based Out-of-distribution Detection, NeurIPS 2020;
+<a href="https://arxiv.org/pdf/2010.03759" class="external-link uri">https://arxiv.org/pdf/2010.03759</a></p>
 <ul>
 <li><p>E(x, y) = energy value</p></li>
 <li><p>if x and y are “compatitble”, lower energy</p></li>
@@ -4933,17 +4971,377 @@ <h1 id="example-2-energy-based-ood-detection">Example 2: Energy-Based OOD Detect
 </li>
 <li><p>With energy scores, ID and OOD distributions become much more
 separable</p></li>
-<li><p>Another “output-based” method like softmax # Conclusion</p></li>
+<li><p>Another “output-based” method like softmax</p></li>
 </ul>
-<div class="codewrapper sourceCode" id="cb15">
+<div class="section level2">
+<h2 id="pytorch-out-of-distribution-detection">PyTorch Out-of-Distribution Detection<a class="anchor" aria-label="anchor" href="#pytorch-out-of-distribution-detection"></a>
+</h2>
+<p>There’s a Pytorch package for OOD detection! <a href="https://pytorch-ood.readthedocs.io/en/latest/info.html" class="external-link uri">https://pytorch-ood.readthedocs.io/en/latest/info.html</a></p>
+<div class="codewrapper sourceCode" id="cb16">
+<h3 class="code-label">PYTHON<i aria-hidden="true" data-feather="chevron-left"></i><i aria-hidden="true" data-feather="chevron-right"></i>
+</h3>
+<pre class="sourceCode python" tabindex="0"><code class="sourceCode python"><span id="cb16-1"><a href="#cb16-1" tabindex="-1"></a><span class="op">!</span>pip install pytorch<span class="op">-</span>ood</span></code></pre>
+</div>
+<div class="section level3">
+<h3 id="energy-based-is-designed-to-work-with-neural-nets-unpack-this-">Energy-based is designed to work with neural nets… unpack this.<a class="anchor" aria-label="anchor" href="#energy-based-is-designed-to-work-with-neural-nets-unpack-this-"></a>
+</h3>
+<p>Let’s train a simple CNN model on the FashionMNIST dataset.</p>
+<div class="codewrapper sourceCode" id="cb17">
+<h3 class="code-label">PYTHON<i aria-hidden="true" data-feather="chevron-left"></i><i aria-hidden="true" data-feather="chevron-right"></i>
+</h3>
+<pre class="sourceCode python" tabindex="0"><code class="sourceCode python"><span id="cb17-1"><a href="#cb17-1" tabindex="-1"></a><span class="im">import</span> torch</span>
+<span id="cb17-2"><a href="#cb17-2" tabindex="-1"></a><span class="im">import</span> torch.nn <span class="im">as</span> nn</span>
+<span id="cb17-3"><a href="#cb17-3" tabindex="-1"></a><span class="im">import</span> torch.optim <span class="im">as</span> optim</span>
+<span id="cb17-4"><a href="#cb17-4" tabindex="-1"></a><span class="im">import</span> torchvision.transforms <span class="im">as</span> transforms</span>
+<span id="cb17-5"><a href="#cb17-5" tabindex="-1"></a><span class="im">from</span> keras.datasets <span class="im">import</span> fashion_mnist</span>
+<span id="cb17-6"><a href="#cb17-6" tabindex="-1"></a><span class="im">import</span> torch.nn.functional <span class="im">as</span> F</span>
+<span id="cb17-7"><a href="#cb17-7" tabindex="-1"></a><span class="im">import</span> matplotlib.pyplot <span class="im">as</span> plt</span>
+<span id="cb17-8"><a href="#cb17-8" tabindex="-1"></a><span class="im">import</span> numpy <span class="im">as</span> np</span>
+<span id="cb17-9"><a href="#cb17-9" tabindex="-1"></a></span>
+<span id="cb17-10"><a href="#cb17-10" tabindex="-1"></a><span class="co"># Load Fashion MNIST dataset</span></span>
+<span id="cb17-11"><a href="#cb17-11" tabindex="-1"></a>(train_images, train_labels), (test_images, test_labels) <span class="op">=</span> fashion_mnist.load_data()</span>
+<span id="cb17-12"><a href="#cb17-12" tabindex="-1"></a></span>
+<span id="cb17-13"><a href="#cb17-13" tabindex="-1"></a><span class="co"># Define classes for simplicity</span></span>
+<span id="cb17-14"><a href="#cb17-14" tabindex="-1"></a>class_names <span class="op">=</span> [<span class="st">'T-shirt/top'</span>, <span class="st">'Trouser'</span>, <span class="st">'Pullover'</span>, <span class="st">'Dress'</span>, <span class="st">'Coat'</span>,</span>
+<span id="cb17-15"><a href="#cb17-15" tabindex="-1"></a>               <span class="st">'Sandal'</span>, <span class="st">'Shirt'</span>, <span class="st">'Sneaker'</span>, <span class="st">'Bag'</span>, <span class="st">'Ankle boot'</span>]</span>
+<span id="cb17-16"><a href="#cb17-16" tabindex="-1"></a></span>
+<span id="cb17-17"><a href="#cb17-17" tabindex="-1"></a><span class="co"># Prepare OOD data - Sandals (5)</span></span>
+<span id="cb17-18"><a href="#cb17-18" tabindex="-1"></a>ood_data <span class="op">=</span> test_images[test_labels <span class="op">==</span> <span class="dv">5</span>]</span>
+<span id="cb17-19"><a href="#cb17-19" tabindex="-1"></a>ood_labels <span class="op">=</span> test_labels[test_labels <span class="op">==</span> <span class="dv">5</span>]</span>
+<span id="cb17-20"><a href="#cb17-20" tabindex="-1"></a><span class="bu">print</span>(<span class="ss">f'ood_data.shape=</span><span class="sc">{</span>ood_data<span class="sc">.</span>shape<span class="sc">}</span><span class="ss">'</span>)</span>
+<span id="cb17-21"><a href="#cb17-21" tabindex="-1"></a></span>
+<span id="cb17-22"><a href="#cb17-22" tabindex="-1"></a><span class="co"># Filter data for T-shirts (0) and Trousers (1) as in-distribution</span></span>
+<span id="cb17-23"><a href="#cb17-23" tabindex="-1"></a>train_filter <span class="op">=</span> np.isin(train_labels, [<span class="dv">0</span>, <span class="dv">1</span>])</span>
+<span id="cb17-24"><a href="#cb17-24" tabindex="-1"></a>test_filter <span class="op">=</span> np.isin(test_labels, [<span class="dv">0</span>, <span class="dv">1</span>])</span>
+<span id="cb17-25"><a href="#cb17-25" tabindex="-1"></a></span>
+<span id="cb17-26"><a href="#cb17-26" tabindex="-1"></a>train_data <span class="op">=</span> train_images[train_filter]</span>
+<span id="cb17-27"><a href="#cb17-27" tabindex="-1"></a>train_labels <span class="op">=</span> train_labels[train_filter]</span>
+<span id="cb17-28"><a href="#cb17-28" tabindex="-1"></a><span class="bu">print</span>(<span class="ss">f'train_data.shape=</span><span class="sc">{</span>train_data<span class="sc">.</span>shape<span class="sc">}</span><span class="ss">'</span>)</span>
+<span id="cb17-29"><a href="#cb17-29" tabindex="-1"></a></span>
+<span id="cb17-30"><a href="#cb17-30" tabindex="-1"></a>test_data <span class="op">=</span> test_images[test_filter]</span>
+<span id="cb17-31"><a href="#cb17-31" tabindex="-1"></a>test_labels <span class="op">=</span> test_labels[test_filter]</span>
+<span id="cb17-32"><a href="#cb17-32" tabindex="-1"></a><span class="bu">print</span>(<span class="ss">f'test_data.shape=</span><span class="sc">{</span>test_data<span class="sc">.</span>shape<span class="sc">}</span><span class="ss">'</span>)</span>
+<span id="cb17-33"><a href="#cb17-33" tabindex="-1"></a></span>
+<span id="cb17-34"><a href="#cb17-34" tabindex="-1"></a><span class="co"># Transform to Tensor and normalize</span></span>
+<span id="cb17-35"><a href="#cb17-35" tabindex="-1"></a>transform <span class="op">=</span> transforms.Compose([</span>
+<span id="cb17-36"><a href="#cb17-36" tabindex="-1"></a>    transforms.ToTensor(),</span>
+<span id="cb17-37"><a href="#cb17-37" tabindex="-1"></a>    transforms.Normalize((<span class="fl">0.5</span>,), (<span class="fl">0.5</span>,))</span>
+<span id="cb17-38"><a href="#cb17-38" tabindex="-1"></a>])</span>
+<span id="cb17-39"><a href="#cb17-39" tabindex="-1"></a></span>
+<span id="cb17-40"><a href="#cb17-40" tabindex="-1"></a><span class="co"># Convert to PyTorch tensors and normalize</span></span>
+<span id="cb17-41"><a href="#cb17-41" tabindex="-1"></a>train_data_tensor <span class="op">=</span> torch.tensor(train_data, dtype<span class="op">=</span>torch.float32).unsqueeze(<span class="dv">1</span>) <span class="op">/</span> <span class="fl">255.0</span></span>
+<span id="cb17-42"><a href="#cb17-42" tabindex="-1"></a>test_data_tensor <span class="op">=</span> torch.tensor(test_data, dtype<span class="op">=</span>torch.float32).unsqueeze(<span class="dv">1</span>) <span class="op">/</span> <span class="fl">255.0</span></span>
+<span id="cb17-43"><a href="#cb17-43" tabindex="-1"></a>ood_data_tensor <span class="op">=</span> torch.tensor(ood_data, dtype<span class="op">=</span>torch.float32).unsqueeze(<span class="dv">1</span>) <span class="op">/</span> <span class="fl">255.0</span></span>
+<span id="cb17-44"><a href="#cb17-44" tabindex="-1"></a></span>
+<span id="cb17-45"><a href="#cb17-45" tabindex="-1"></a>train_labels_tensor <span class="op">=</span> torch.tensor(train_labels, dtype<span class="op">=</span>torch.<span class="bu">long</span>)</span>
+<span id="cb17-46"><a href="#cb17-46" tabindex="-1"></a>test_labels_tensor <span class="op">=</span> torch.tensor(test_labels, dtype<span class="op">=</span>torch.<span class="bu">long</span>)</span>
+<span id="cb17-47"><a href="#cb17-47" tabindex="-1"></a></span>
+<span id="cb17-48"><a href="#cb17-48" tabindex="-1"></a>train_dataset <span class="op">=</span> torch.utils.data.TensorDataset(train_data_tensor, train_labels_tensor)</span>
+<span id="cb17-49"><a href="#cb17-49" tabindex="-1"></a>test_dataset <span class="op">=</span> torch.utils.data.TensorDataset(test_data_tensor, test_labels_tensor)</span>
+<span id="cb17-50"><a href="#cb17-50" tabindex="-1"></a>ood_dataset <span class="op">=</span> torch.utils.data.TensorDataset(ood_data_tensor, torch.zeros(ood_data_tensor.shape[<span class="dv">0</span>], dtype<span class="op">=</span>torch.<span class="bu">long</span>))</span>
+<span id="cb17-51"><a href="#cb17-51" tabindex="-1"></a></span>
+<span id="cb17-52"><a href="#cb17-52" tabindex="-1"></a>train_loader <span class="op">=</span> torch.utils.data.DataLoader(train_dataset, batch_size<span class="op">=</span><span class="dv">64</span>, shuffle<span class="op">=</span><span class="va">True</span>)</span>
+<span id="cb17-53"><a href="#cb17-53" tabindex="-1"></a>test_loader <span class="op">=</span> torch.utils.data.DataLoader(test_dataset, batch_size<span class="op">=</span><span class="dv">64</span>, shuffle<span class="op">=</span><span class="va">False</span>)</span>
+<span id="cb17-54"><a href="#cb17-54" tabindex="-1"></a>ood_loader <span class="op">=</span> torch.utils.data.DataLoader(ood_dataset, batch_size<span class="op">=</span><span class="dv">64</span>, shuffle<span class="op">=</span><span class="va">False</span>)</span>
+<span id="cb17-55"><a href="#cb17-55" tabindex="-1"></a></span>
+<span id="cb17-56"><a href="#cb17-56" tabindex="-1"></a><span class="co"># Define a simple CNN model</span></span>
+<span id="cb17-57"><a href="#cb17-57" tabindex="-1"></a><span class="kw">class</span> SimpleCNN(nn.Module):</span>
+<span id="cb17-58"><a href="#cb17-58" tabindex="-1"></a>    <span class="kw">def</span> <span class="fu">__init__</span>(<span class="va">self</span>):</span>
+<span id="cb17-59"><a href="#cb17-59" tabindex="-1"></a>        <span class="bu">super</span>(SimpleCNN, <span class="va">self</span>).<span class="fu">__init__</span>()</span>
+<span id="cb17-60"><a href="#cb17-60" tabindex="-1"></a>        <span class="va">self</span>.conv1 <span class="op">=</span> nn.Conv2d(<span class="dv">1</span>, <span class="dv">32</span>, kernel_size<span class="op">=</span><span class="dv">3</span>)</span>
+<span id="cb17-61"><a href="#cb17-61" tabindex="-1"></a>        <span class="va">self</span>.conv2 <span class="op">=</span> nn.Conv2d(<span class="dv">32</span>, <span class="dv">64</span>, kernel_size<span class="op">=</span><span class="dv">3</span>)</span>
+<span id="cb17-62"><a href="#cb17-62" tabindex="-1"></a>        <span class="va">self</span>.fc1 <span class="op">=</span> nn.Linear(<span class="dv">64</span><span class="op">*</span><span class="dv">5</span><span class="op">*</span><span class="dv">5</span>, <span class="dv">128</span>)  <span class="co"># Updated this line</span></span>
+<span id="cb17-63"><a href="#cb17-63" tabindex="-1"></a>        <span class="va">self</span>.fc2 <span class="op">=</span> nn.Linear(<span class="dv">128</span>, <span class="dv">2</span>)</span>
+<span id="cb17-64"><a href="#cb17-64" tabindex="-1"></a></span>
+<span id="cb17-65"><a href="#cb17-65" tabindex="-1"></a>    <span class="kw">def</span> forward(<span class="va">self</span>, x):</span>
+<span id="cb17-66"><a href="#cb17-66" tabindex="-1"></a>        x <span class="op">=</span> F.relu(F.max_pool2d(<span class="va">self</span>.conv1(x), <span class="dv">2</span>))</span>
+<span id="cb17-67"><a href="#cb17-67" tabindex="-1"></a>        x <span class="op">=</span> F.relu(F.max_pool2d(<span class="va">self</span>.conv2(x), <span class="dv">2</span>))</span>
+<span id="cb17-68"><a href="#cb17-68" tabindex="-1"></a>        x <span class="op">=</span> x.view(<span class="op">-</span><span class="dv">1</span>, <span class="dv">64</span><span class="op">*</span><span class="dv">5</span><span class="op">*</span><span class="dv">5</span>)  <span class="co"># Updated this line</span></span>
+<span id="cb17-69"><a href="#cb17-69" tabindex="-1"></a>        x <span class="op">=</span> F.relu(<span class="va">self</span>.fc1(x))</span>
+<span id="cb17-70"><a href="#cb17-70" tabindex="-1"></a>        x <span class="op">=</span> <span class="va">self</span>.fc2(x)</span>
+<span id="cb17-71"><a href="#cb17-71" tabindex="-1"></a>        <span class="cf">return</span> x</span>
+<span id="cb17-72"><a href="#cb17-72" tabindex="-1"></a></span>
+<span id="cb17-73"><a href="#cb17-73" tabindex="-1"></a>device <span class="op">=</span> torch.device(<span class="st">'cuda'</span> <span class="cf">if</span> torch.cuda.is_available() <span class="cf">else</span> <span class="st">'cpu'</span>)</span>
+<span id="cb17-74"><a href="#cb17-74" tabindex="-1"></a>model <span class="op">=</span> SimpleCNN().to(device)</span>
+<span id="cb17-75"><a href="#cb17-75" tabindex="-1"></a>criterion <span class="op">=</span> nn.CrossEntropyLoss()</span>
+<span id="cb17-76"><a href="#cb17-76" tabindex="-1"></a>optimizer <span class="op">=</span> optim.Adam(model.parameters(), lr<span class="op">=</span><span class="fl">0.001</span>)</span>
+<span id="cb17-77"><a href="#cb17-77" tabindex="-1"></a></span>
+<span id="cb17-78"><a href="#cb17-78" tabindex="-1"></a><span class="kw">def</span> train_model(model, train_loader, criterion, optimizer, epochs<span class="op">=</span><span class="dv">5</span>):</span>
+<span id="cb17-79"><a href="#cb17-79" tabindex="-1"></a>    model.train()</span>
+<span id="cb17-80"><a href="#cb17-80" tabindex="-1"></a>    <span class="cf">for</span> epoch <span class="kw">in</span> <span class="bu">range</span>(epochs):</span>
+<span id="cb17-81"><a href="#cb17-81" tabindex="-1"></a>        running_loss <span class="op">=</span> <span class="fl">0.0</span></span>
+<span id="cb17-82"><a href="#cb17-82" tabindex="-1"></a>        <span class="cf">for</span> inputs, labels <span class="kw">in</span> train_loader:</span>
+<span id="cb17-83"><a href="#cb17-83" tabindex="-1"></a>            inputs, labels <span class="op">=</span> inputs.to(device), labels.to(device)</span>
+<span id="cb17-84"><a href="#cb17-84" tabindex="-1"></a>            optimizer.zero_grad()</span>
+<span id="cb17-85"><a href="#cb17-85" tabindex="-1"></a>            outputs <span class="op">=</span> model(inputs)</span>
+<span id="cb17-86"><a href="#cb17-86" tabindex="-1"></a>            loss <span class="op">=</span> criterion(outputs, labels)</span>
+<span id="cb17-87"><a href="#cb17-87" tabindex="-1"></a>            loss.backward()</span>
+<span id="cb17-88"><a href="#cb17-88" tabindex="-1"></a>            optimizer.step()</span>
+<span id="cb17-89"><a href="#cb17-89" tabindex="-1"></a>            running_loss <span class="op">+=</span> loss.item()</span>
+<span id="cb17-90"><a href="#cb17-90" tabindex="-1"></a>        <span class="bu">print</span>(<span class="ss">f'Epoch </span><span class="sc">{</span>epoch<span class="op">+</span><span class="dv">1</span><span class="sc">}</span><span class="ss">, Loss: </span><span class="sc">{</span>running_loss<span class="op">/</span><span class="bu">len</span>(train_loader)<span class="sc">}</span><span class="ss">'</span>)</span>
+<span id="cb17-91"><a href="#cb17-91" tabindex="-1"></a></span>
+<span id="cb17-92"><a href="#cb17-92" tabindex="-1"></a>train_model(model, train_loader, criterion, optimizer)</span></code></pre>
+</div>
+<div class="codewrapper sourceCode" id="cb18">
+<h3 class="code-label">PYTHON<i aria-hidden="true" data-feather="chevron-left"></i><i aria-hidden="true" data-feather="chevron-right"></i>
+</h3>
+<pre class="sourceCode python" tabindex="0"><code class="sourceCode python"><span id="cb18-1"><a href="#cb18-1" tabindex="-1"></a><span class="im">from</span> sklearn.metrics <span class="im">import</span> confusion_matrix, ConfusionMatrixDisplay</span>
+<span id="cb18-2"><a href="#cb18-2" tabindex="-1"></a></span>
+<span id="cb18-3"><a href="#cb18-3" tabindex="-1"></a><span class="co"># Function to plot confusion matrix</span></span>
+<span id="cb18-4"><a href="#cb18-4" tabindex="-1"></a><span class="kw">def</span> plot_confusion_matrix(labels, predictions, title):</span>
+<span id="cb18-5"><a href="#cb18-5" tabindex="-1"></a>    cm <span class="op">=</span> confusion_matrix(labels, predictions, labels<span class="op">=</span>[<span class="dv">0</span>, <span class="dv">1</span>])</span>
+<span id="cb18-6"><a href="#cb18-6" tabindex="-1"></a>    disp <span class="op">=</span> ConfusionMatrixDisplay(confusion_matrix<span class="op">=</span>cm, display_labels<span class="op">=</span>[<span class="st">"T-shirt/top"</span>, <span class="st">"Trouser"</span>])</span>
+<span id="cb18-7"><a href="#cb18-7" tabindex="-1"></a>    disp.plot(cmap<span class="op">=</span>plt.cm.Blues)</span>
+<span id="cb18-8"><a href="#cb18-8" tabindex="-1"></a>    plt.title(title)</span>
+<span id="cb18-9"><a href="#cb18-9" tabindex="-1"></a>    plt.show()</span>
+<span id="cb18-10"><a href="#cb18-10" tabindex="-1"></a></span>
+<span id="cb18-11"><a href="#cb18-11" tabindex="-1"></a><span class="co"># Function to evaluate model on a dataset</span></span>
+<span id="cb18-12"><a href="#cb18-12" tabindex="-1"></a><span class="kw">def</span> evaluate_model(model, dataloader, device):</span>
+<span id="cb18-13"><a href="#cb18-13" tabindex="-1"></a>    model.<span class="bu">eval</span>()</span>
+<span id="cb18-14"><a href="#cb18-14" tabindex="-1"></a>    all_labels <span class="op">=</span> []</span>
+<span id="cb18-15"><a href="#cb18-15" tabindex="-1"></a>    all_predictions <span class="op">=</span> []</span>
+<span id="cb18-16"><a href="#cb18-16" tabindex="-1"></a>    <span class="cf">with</span> torch.no_grad():</span>
+<span id="cb18-17"><a href="#cb18-17" tabindex="-1"></a>        <span class="cf">for</span> inputs, labels <span class="kw">in</span> dataloader:</span>
+<span id="cb18-18"><a href="#cb18-18" tabindex="-1"></a>            inputs, labels <span class="op">=</span> inputs.to(device), labels.to(device)</span>
+<span id="cb18-19"><a href="#cb18-19" tabindex="-1"></a>            outputs <span class="op">=</span> model(inputs)</span>
+<span id="cb18-20"><a href="#cb18-20" tabindex="-1"></a>            _, preds <span class="op">=</span> torch.<span class="bu">max</span>(outputs, <span class="dv">1</span>)</span>
+<span id="cb18-21"><a href="#cb18-21" tabindex="-1"></a>            all_labels.extend(labels.cpu().numpy())</span>
+<span id="cb18-22"><a href="#cb18-22" tabindex="-1"></a>            all_predictions.extend(preds.cpu().numpy())</span>
+<span id="cb18-23"><a href="#cb18-23" tabindex="-1"></a>    <span class="cf">return</span> np.array(all_labels), np.array(all_predictions)</span>
+<span id="cb18-24"><a href="#cb18-24" tabindex="-1"></a></span>
+<span id="cb18-25"><a href="#cb18-25" tabindex="-1"></a><span class="co"># Evaluate on train data</span></span>
+<span id="cb18-26"><a href="#cb18-26" tabindex="-1"></a>train_labels, train_predictions <span class="op">=</span> evaluate_model(model, train_loader, device)</span>
+<span id="cb18-27"><a href="#cb18-27" tabindex="-1"></a>plot_confusion_matrix(train_labels, train_predictions, <span class="st">"Confusion Matrix for Train Data"</span>)</span>
+<span id="cb18-28"><a href="#cb18-28" tabindex="-1"></a></span>
+<span id="cb18-29"><a href="#cb18-29" tabindex="-1"></a><span class="co"># Evaluate on test data</span></span>
+<span id="cb18-30"><a href="#cb18-30" tabindex="-1"></a>test_labels, test_predictions <span class="op">=</span> evaluate_model(model, test_loader, device)</span>
+<span id="cb18-31"><a href="#cb18-31" tabindex="-1"></a>plot_confusion_matrix(test_labels, test_predictions, <span class="st">"Confusion Matrix for Test Data"</span>)</span></code></pre>
+</div>
+<div class="codewrapper sourceCode" id="cb19">
+<h3 class="code-label">PYTHON<i aria-hidden="true" data-feather="chevron-left"></i><i aria-hidden="true" data-feather="chevron-right"></i>
+</h3>
+<pre class="sourceCode python" tabindex="0"><code class="sourceCode python"><span id="cb19-1"><a href="#cb19-1" tabindex="-1"></a><span class="im">from</span> scipy.stats <span class="im">import</span> gaussian_kde</span>
+<span id="cb19-2"><a href="#cb19-2" tabindex="-1"></a><span class="im">from</span> pytorch_ood.detector <span class="im">import</span> EnergyBased</span>
+<span id="cb19-3"><a href="#cb19-3" tabindex="-1"></a><span class="im">from</span> sklearn.metrics <span class="im">import</span> precision_recall_fscore_support, accuracy_score</span>
+<span id="cb19-4"><a href="#cb19-4" tabindex="-1"></a></span>
+<span id="cb19-5"><a href="#cb19-5" tabindex="-1"></a><span class="co"># Compute softmax scores</span></span>
+<span id="cb19-6"><a href="#cb19-6" tabindex="-1"></a><span class="kw">def</span> get_softmax_scores(model, dataloader):</span>
+<span id="cb19-7"><a href="#cb19-7" tabindex="-1"></a>    model.<span class="bu">eval</span>()</span>
+<span id="cb19-8"><a href="#cb19-8" tabindex="-1"></a>    softmax_scores <span class="op">=</span> []</span>
+<span id="cb19-9"><a href="#cb19-9" tabindex="-1"></a>    <span class="cf">with</span> torch.no_grad():</span>
+<span id="cb19-10"><a href="#cb19-10" tabindex="-1"></a>        <span class="cf">for</span> inputs, _ <span class="kw">in</span> dataloader:</span>
+<span id="cb19-11"><a href="#cb19-11" tabindex="-1"></a>            inputs <span class="op">=</span> inputs.to(device)</span>
+<span id="cb19-12"><a href="#cb19-12" tabindex="-1"></a>            outputs <span class="op">=</span> model(inputs)</span>
+<span id="cb19-13"><a href="#cb19-13" tabindex="-1"></a>            softmax <span class="op">=</span> torch.nn.functional.softmax(outputs, dim<span class="op">=</span><span class="dv">1</span>)</span>
+<span id="cb19-14"><a href="#cb19-14" tabindex="-1"></a>            softmax_scores.extend(softmax.cpu().numpy())</span>
+<span id="cb19-15"><a href="#cb19-15" tabindex="-1"></a>    <span class="cf">return</span> np.array(softmax_scores)</span>
+<span id="cb19-16"><a href="#cb19-16" tabindex="-1"></a></span>
+<span id="cb19-17"><a href="#cb19-17" tabindex="-1"></a>id_softmax_scores <span class="op">=</span> get_softmax_scores(model, test_loader)</span>
+<span id="cb19-18"><a href="#cb19-18" tabindex="-1"></a>ood_softmax_scores <span class="op">=</span> get_softmax_scores(model, ood_loader)</span>
+<span id="cb19-19"><a href="#cb19-19" tabindex="-1"></a></span>
+<span id="cb19-20"><a href="#cb19-20" tabindex="-1"></a><span class="co"># Initialize the energy-based OOD detector</span></span>
+<span id="cb19-21"><a href="#cb19-21" tabindex="-1"></a>energy_detector <span class="op">=</span> EnergyBased(model, t<span class="op">=</span><span class="fl">1.0</span>)</span>
+<span id="cb19-22"><a href="#cb19-22" tabindex="-1"></a></span>
+<span id="cb19-23"><a href="#cb19-23" tabindex="-1"></a><span class="co"># Compute energy scores</span></span>
+<span id="cb19-24"><a href="#cb19-24" tabindex="-1"></a><span class="kw">def</span> get_energy_scores(detector, dataloader):</span>
+<span id="cb19-25"><a href="#cb19-25" tabindex="-1"></a>    scores <span class="op">=</span> []</span>
+<span id="cb19-26"><a href="#cb19-26" tabindex="-1"></a>    detector.model.<span class="bu">eval</span>()</span>
+<span id="cb19-27"><a href="#cb19-27" tabindex="-1"></a>    <span class="cf">with</span> torch.no_grad():</span>
+<span id="cb19-28"><a href="#cb19-28" tabindex="-1"></a>        <span class="cf">for</span> inputs, _ <span class="kw">in</span> dataloader:</span>
+<span id="cb19-29"><a href="#cb19-29" tabindex="-1"></a>            inputs <span class="op">=</span> inputs.to(device)</span>
+<span id="cb19-30"><a href="#cb19-30" tabindex="-1"></a>            score <span class="op">=</span> detector.predict(inputs)</span>
+<span id="cb19-31"><a href="#cb19-31" tabindex="-1"></a>            scores.extend(score.cpu().numpy())</span>
+<span id="cb19-32"><a href="#cb19-32" tabindex="-1"></a>    <span class="cf">return</span> np.array(scores)</span>
+<span id="cb19-33"><a href="#cb19-33" tabindex="-1"></a></span>
+<span id="cb19-34"><a href="#cb19-34" tabindex="-1"></a>id_energy_scores <span class="op">=</span> get_energy_scores(energy_detector, test_loader)</span>
+<span id="cb19-35"><a href="#cb19-35" tabindex="-1"></a>ood_energy_scores <span class="op">=</span> get_energy_scores(energy_detector, ood_loader)</span>
+<span id="cb19-36"><a href="#cb19-36" tabindex="-1"></a></span>
+<span id="cb19-37"><a href="#cb19-37" tabindex="-1"></a><span class="im">import</span> matplotlib.pyplot <span class="im">as</span> plt</span>
+<span id="cb19-38"><a href="#cb19-38" tabindex="-1"></a></span>
+<span id="cb19-39"><a href="#cb19-39" tabindex="-1"></a></span>
+<span id="cb19-40"><a href="#cb19-40" tabindex="-1"></a><span class="co"># Plot PSDs</span></span>
+<span id="cb19-41"><a href="#cb19-41" tabindex="-1"></a></span>
+<span id="cb19-42"><a href="#cb19-42" tabindex="-1"></a><span class="co"># Function to plot PSD</span></span>
+<span id="cb19-43"><a href="#cb19-43" tabindex="-1"></a><span class="kw">def</span> plot_psd(id_scores, ood_scores, method_name):</span>
+<span id="cb19-44"><a href="#cb19-44" tabindex="-1"></a>    plt.figure(figsize<span class="op">=</span>(<span class="dv">12</span>, <span class="dv">6</span>))</span>
+<span id="cb19-45"><a href="#cb19-45" tabindex="-1"></a>    alpha <span class="op">=</span> <span class="fl">0.3</span></span>
+<span id="cb19-46"><a href="#cb19-46" tabindex="-1"></a></span>
+<span id="cb19-47"><a href="#cb19-47" tabindex="-1"></a>    <span class="co"># Plot PSD for ID scores</span></span>
+<span id="cb19-48"><a href="#cb19-48" tabindex="-1"></a>    id_density <span class="op">=</span> gaussian_kde(id_scores)</span>
+<span id="cb19-49"><a href="#cb19-49" tabindex="-1"></a>    x_id <span class="op">=</span> np.linspace(id_scores.<span class="bu">min</span>(), id_scores.<span class="bu">max</span>(), <span class="dv">1000</span>)</span>
+<span id="cb19-50"><a href="#cb19-50" tabindex="-1"></a>    plt.plot(x_id, id_density(x_id), label<span class="op">=</span><span class="ss">f'ID (</span><span class="sc">{</span>method_name<span class="sc">}</span><span class="ss">)'</span>, color<span class="op">=</span><span class="st">'blue'</span>, alpha<span class="op">=</span>alpha)</span>
+<span id="cb19-51"><a href="#cb19-51" tabindex="-1"></a></span>
+<span id="cb19-52"><a href="#cb19-52" tabindex="-1"></a>    <span class="co"># Plot PSD for OOD scores</span></span>
+<span id="cb19-53"><a href="#cb19-53" tabindex="-1"></a>    ood_density <span class="op">=</span> gaussian_kde(ood_scores)</span>
+<span id="cb19-54"><a href="#cb19-54" tabindex="-1"></a>    x_ood <span class="op">=</span> np.linspace(ood_scores.<span class="bu">min</span>(), ood_scores.<span class="bu">max</span>(), <span class="dv">1000</span>)</span>
+<span id="cb19-55"><a href="#cb19-55" tabindex="-1"></a>    plt.plot(x_ood, ood_density(x_ood), label<span class="op">=</span><span class="ss">f'OOD (</span><span class="sc">{</span>method_name<span class="sc">}</span><span class="ss">)'</span>, color<span class="op">=</span><span class="st">'red'</span>, alpha<span class="op">=</span>alpha)</span>
+<span id="cb19-56"><a href="#cb19-56" tabindex="-1"></a></span>
+<span id="cb19-57"><a href="#cb19-57" tabindex="-1"></a>    plt.xlabel(<span class="st">'Score'</span>)</span>
+<span id="cb19-58"><a href="#cb19-58" tabindex="-1"></a>    plt.ylabel(<span class="st">'Density'</span>)</span>
+<span id="cb19-59"><a href="#cb19-59" tabindex="-1"></a>    plt.title(<span class="ss">f'Probability Density Distributions for </span><span class="sc">{</span>method_name<span class="sc">}</span><span class="ss"> Scores'</span>)</span>
+<span id="cb19-60"><a href="#cb19-60" tabindex="-1"></a>    plt.legend()</span>
+<span id="cb19-61"><a href="#cb19-61" tabindex="-1"></a>    plt.show()</span>
+<span id="cb19-62"><a href="#cb19-62" tabindex="-1"></a></span>
+<span id="cb19-63"><a href="#cb19-63" tabindex="-1"></a><span class="co"># Plot PSD for softmax scores</span></span>
+<span id="cb19-64"><a href="#cb19-64" tabindex="-1"></a>plot_psd(id_softmax_scores[:, <span class="dv">1</span>], ood_softmax_scores[:, <span class="dv">1</span>], <span class="st">'Softmax'</span>)</span>
+<span id="cb19-65"><a href="#cb19-65" tabindex="-1"></a></span>
+<span id="cb19-66"><a href="#cb19-66" tabindex="-1"></a><span class="co"># Plot PSD for energy scores</span></span>
+<span id="cb19-67"><a href="#cb19-67" tabindex="-1"></a>plot_psd(id_energy_scores, ood_energy_scores, <span class="st">'Energy'</span>)</span>
+<span id="cb19-68"><a href="#cb19-68" tabindex="-1"></a></span></code></pre>
+</div>
+<div class="codewrapper sourceCode" id="cb20">
+<h3 class="code-label">PYTHON<i aria-hidden="true" data-feather="chevron-left"></i><i aria-hidden="true" data-feather="chevron-right"></i>
+</h3>
+<pre class="sourceCode python" tabindex="0"><code class="sourceCode python"><span id="cb20-1"><a href="#cb20-1" tabindex="-1"></a><span class="im">import</span> numpy <span class="im">as</span> np</span>
+<span id="cb20-2"><a href="#cb20-2" tabindex="-1"></a><span class="im">import</span> matplotlib.pyplot <span class="im">as</span> plt</span>
+<span id="cb20-3"><a href="#cb20-3" tabindex="-1"></a><span class="im">from</span> sklearn.metrics <span class="im">import</span> precision_recall_fscore_support, accuracy_score, confusion_matrix, ConfusionMatrixDisplay</span>
+<span id="cb20-4"><a href="#cb20-4" tabindex="-1"></a></span>
+<span id="cb20-5"><a href="#cb20-5" tabindex="-1"></a><span class="co"># Define thresholds to evaluate</span></span>
+<span id="cb20-6"><a href="#cb20-6" tabindex="-1"></a>thresholds <span class="op">=</span> np.linspace(id_energy_scores.<span class="bu">min</span>(), id_energy_scores.<span class="bu">max</span>(), <span class="dv">50</span>)</span>
+<span id="cb20-7"><a href="#cb20-7" tabindex="-1"></a></span>
+<span id="cb20-8"><a href="#cb20-8" tabindex="-1"></a><span class="co"># Store evaluation metrics for each threshold</span></span>
+<span id="cb20-9"><a href="#cb20-9" tabindex="-1"></a>accuracies <span class="op">=</span> []</span>
+<span id="cb20-10"><a href="#cb20-10" tabindex="-1"></a>precisions <span class="op">=</span> []</span>
+<span id="cb20-11"><a href="#cb20-11" tabindex="-1"></a>recalls <span class="op">=</span> []</span>
+<span id="cb20-12"><a href="#cb20-12" tabindex="-1"></a>f1_scores <span class="op">=</span> []</span>
+<span id="cb20-13"><a href="#cb20-13" tabindex="-1"></a></span>
+<span id="cb20-14"><a href="#cb20-14" tabindex="-1"></a><span class="co"># True labels for OOD data (since they are not part of the original labels)</span></span>
+<span id="cb20-15"><a href="#cb20-15" tabindex="-1"></a>ood_true_labels <span class="op">=</span> np.full(<span class="bu">len</span>(ood_energy_scores), <span class="op">-</span><span class="dv">1</span>)</span>
+<span id="cb20-16"><a href="#cb20-16" tabindex="-1"></a></span>
+<span id="cb20-17"><a href="#cb20-17" tabindex="-1"></a><span class="co"># We need the test_labels to be aligned with the ID data</span></span>
+<span id="cb20-18"><a href="#cb20-18" tabindex="-1"></a>id_true_labels <span class="op">=</span> test_labels[:<span class="bu">len</span>(id_energy_scores)]</span>
+<span id="cb20-19"><a href="#cb20-19" tabindex="-1"></a></span>
+<span id="cb20-20"><a href="#cb20-20" tabindex="-1"></a><span class="cf">for</span> threshold <span class="kw">in</span> thresholds:</span>
+<span id="cb20-21"><a href="#cb20-21" tabindex="-1"></a>    <span class="co"># Classify OOD examples based on energy scores</span></span>
+<span id="cb20-22"><a href="#cb20-22" tabindex="-1"></a>    ood_classifications <span class="op">=</span> np.where(ood_energy_scores <span class="op">&gt;=</span> threshold, <span class="op">-</span><span class="dv">1</span>,  <span class="co"># classified as OOD</span></span>
+<span id="cb20-23"><a href="#cb20-23" tabindex="-1"></a>                                   np.where(ood_energy_scores <span class="op">&lt;</span> threshold, <span class="dv">0</span>, <span class="op">-</span><span class="dv">1</span>))  <span class="co"># classified as ID</span></span>
+<span id="cb20-24"><a href="#cb20-24" tabindex="-1"></a></span>
+<span id="cb20-25"><a href="#cb20-25" tabindex="-1"></a>    <span class="co"># Classify ID examples based on energy scores</span></span>
+<span id="cb20-26"><a href="#cb20-26" tabindex="-1"></a>    id_classifications <span class="op">=</span> np.where(id_energy_scores <span class="op">&gt;=</span> threshold, <span class="op">-</span><span class="dv">1</span>,  <span class="co"># classified as OOD</span></span>
+<span id="cb20-27"><a href="#cb20-27" tabindex="-1"></a>                                  np.where(id_energy_scores <span class="op">&lt;</span> threshold, id_true_labels, <span class="op">-</span><span class="dv">1</span>))  <span class="co"># classified as ID</span></span>
+<span id="cb20-28"><a href="#cb20-28" tabindex="-1"></a></span>
+<span id="cb20-29"><a href="#cb20-29" tabindex="-1"></a>    <span class="co"># Combine OOD and ID classifications and true labels</span></span>
+<span id="cb20-30"><a href="#cb20-30" tabindex="-1"></a>    all_predictions <span class="op">=</span> np.concatenate([ood_classifications, id_classifications])</span>
+<span id="cb20-31"><a href="#cb20-31" tabindex="-1"></a>    all_true_labels <span class="op">=</span> np.concatenate([ood_true_labels, id_true_labels])</span>
+<span id="cb20-32"><a href="#cb20-32" tabindex="-1"></a></span>
+<span id="cb20-33"><a href="#cb20-33" tabindex="-1"></a>    <span class="co"># Evaluate metrics</span></span>
+<span id="cb20-34"><a href="#cb20-34" tabindex="-1"></a>    precision, recall, f1, _ <span class="op">=</span> precision_recall_fscore_support(all_true_labels, all_predictions, labels<span class="op">=</span>[<span class="dv">0</span>, <span class="dv">1</span>], average<span class="op">=</span><span class="st">'macro'</span>, zero_division<span class="op">=</span><span class="dv">0</span>)</span>
+<span id="cb20-35"><a href="#cb20-35" tabindex="-1"></a>    accuracy <span class="op">=</span> accuracy_score(all_true_labels, all_predictions)</span>
+<span id="cb20-36"><a href="#cb20-36" tabindex="-1"></a></span>
+<span id="cb20-37"><a href="#cb20-37" tabindex="-1"></a>    accuracies.append(accuracy)</span>
+<span id="cb20-38"><a href="#cb20-38" tabindex="-1"></a>    precisions.append(precision)</span>
+<span id="cb20-39"><a href="#cb20-39" tabindex="-1"></a>    recalls.append(recall)</span>
+<span id="cb20-40"><a href="#cb20-40" tabindex="-1"></a>    f1_scores.append(f1)</span>
+<span id="cb20-41"><a href="#cb20-41" tabindex="-1"></a></span>
+<span id="cb20-42"><a href="#cb20-42" tabindex="-1"></a><span class="co"># Find the best thresholds for each metric</span></span>
+<span id="cb20-43"><a href="#cb20-43" tabindex="-1"></a>best_f1_index <span class="op">=</span> np.argmax(f1_scores)</span>
+<span id="cb20-44"><a href="#cb20-44" tabindex="-1"></a>best_f1_threshold <span class="op">=</span> thresholds[best_f1_index]</span>
+<span id="cb20-45"><a href="#cb20-45" tabindex="-1"></a></span>
+<span id="cb20-46"><a href="#cb20-46" tabindex="-1"></a>best_precision_index <span class="op">=</span> np.argmax(precisions)</span>
+<span id="cb20-47"><a href="#cb20-47" tabindex="-1"></a>best_precision_threshold <span class="op">=</span> thresholds[best_precision_index]</span>
+<span id="cb20-48"><a href="#cb20-48" tabindex="-1"></a></span>
+<span id="cb20-49"><a href="#cb20-49" tabindex="-1"></a>best_recall_index <span class="op">=</span> np.argmax(recalls)</span>
+<span id="cb20-50"><a href="#cb20-50" tabindex="-1"></a>best_recall_threshold <span class="op">=</span> thresholds[best_recall_index]</span>
+<span id="cb20-51"><a href="#cb20-51" tabindex="-1"></a></span>
+<span id="cb20-52"><a href="#cb20-52" tabindex="-1"></a><span class="bu">print</span>(<span class="ss">f"Best F1 threshold: </span><span class="sc">{</span>best_f1_threshold<span class="sc">}</span><span class="ss">, F1 Score: </span><span class="sc">{</span>f1_scores[best_f1_index]<span class="sc">}</span><span class="ss">"</span>)</span>
+<span id="cb20-53"><a href="#cb20-53" tabindex="-1"></a><span class="bu">print</span>(<span class="ss">f"Best Precision threshold: </span><span class="sc">{</span>best_precision_threshold<span class="sc">}</span><span class="ss">, Precision: </span><span class="sc">{</span>precisions[best_precision_index]<span class="sc">}</span><span class="ss">"</span>)</span>
+<span id="cb20-54"><a href="#cb20-54" tabindex="-1"></a><span class="bu">print</span>(<span class="ss">f"Best Recall threshold: </span><span class="sc">{</span>best_recall_threshold<span class="sc">}</span><span class="ss">, Recall: </span><span class="sc">{</span>recalls[best_recall_index]<span class="sc">}</span><span class="ss">"</span>)</span>
+<span id="cb20-55"><a href="#cb20-55" tabindex="-1"></a></span>
+<span id="cb20-56"><a href="#cb20-56" tabindex="-1"></a><span class="co"># Plot metrics as functions of the threshold</span></span>
+<span id="cb20-57"><a href="#cb20-57" tabindex="-1"></a>plt.figure(figsize<span class="op">=</span>(<span class="dv">12</span>, <span class="dv">8</span>))</span>
+<span id="cb20-58"><a href="#cb20-58" tabindex="-1"></a>plt.plot(thresholds, precisions, label<span class="op">=</span><span class="st">'Precision'</span>, color<span class="op">=</span><span class="st">'g'</span>)</span>
+<span id="cb20-59"><a href="#cb20-59" tabindex="-1"></a>plt.plot(thresholds, recalls, label<span class="op">=</span><span class="st">'Recall'</span>, color<span class="op">=</span><span class="st">'b'</span>)</span>
+<span id="cb20-60"><a href="#cb20-60" tabindex="-1"></a>plt.plot(thresholds, f1_scores, label<span class="op">=</span><span class="st">'F1 Score'</span>, color<span class="op">=</span><span class="st">'r'</span>)</span>
+<span id="cb20-61"><a href="#cb20-61" tabindex="-1"></a></span>
+<span id="cb20-62"><a href="#cb20-62" tabindex="-1"></a><span class="co"># Add best threshold indicators</span></span>
+<span id="cb20-63"><a href="#cb20-63" tabindex="-1"></a>plt.axvline(x<span class="op">=</span>best_f1_threshold, color<span class="op">=</span><span class="st">'r'</span>, linestyle<span class="op">=</span><span class="st">'--'</span>, label<span class="op">=</span><span class="ss">f'Best F1 Threshold: </span><span class="sc">{</span>best_f1_threshold<span class="sc">:.2f}</span><span class="ss">'</span>)</span>
+<span id="cb20-64"><a href="#cb20-64" tabindex="-1"></a>plt.axvline(x<span class="op">=</span>best_precision_threshold, color<span class="op">=</span><span class="st">'g'</span>, linestyle<span class="op">=</span><span class="st">'--'</span>, label<span class="op">=</span><span class="ss">f'Best Precision Threshold: </span><span class="sc">{</span>best_precision_threshold<span class="sc">:.2f}</span><span class="ss">'</span>)</span>
+<span id="cb20-65"><a href="#cb20-65" tabindex="-1"></a>plt.axvline(x<span class="op">=</span>best_recall_threshold, color<span class="op">=</span><span class="st">'b'</span>, linestyle<span class="op">=</span><span class="st">'--'</span>, label<span class="op">=</span><span class="ss">f'Best Recall Threshold: </span><span class="sc">{</span>best_recall_threshold<span class="sc">:.2f}</span><span class="ss">'</span>)</span>
+<span id="cb20-66"><a href="#cb20-66" tabindex="-1"></a></span>
+<span id="cb20-67"><a href="#cb20-67" tabindex="-1"></a>plt.xlabel(<span class="st">'Threshold'</span>)</span>
+<span id="cb20-68"><a href="#cb20-68" tabindex="-1"></a>plt.ylabel(<span class="st">'Metric Value'</span>)</span>
+<span id="cb20-69"><a href="#cb20-69" tabindex="-1"></a>plt.title(<span class="st">'Evaluation Metrics as Functions of Threshold (Energy-Based OOD Detection)'</span>)</span>
+<span id="cb20-70"><a href="#cb20-70" tabindex="-1"></a>plt.legend()</span>
+<span id="cb20-71"><a href="#cb20-71" tabindex="-1"></a>plt.show()</span></code></pre>
+</div>
+<div class="codewrapper sourceCode" id="cb21">
+<h3 class="code-label">PYTHON<i aria-hidden="true" data-feather="chevron-left"></i><i aria-hidden="true" data-feather="chevron-right"></i>
+</h3>
+<pre class="sourceCode python" tabindex="0"><code class="sourceCode python"><span id="cb21-1"><a href="#cb21-1" tabindex="-1"></a><span class="im">import</span> numpy <span class="im">as</span> np</span>
+<span id="cb21-2"><a href="#cb21-2" tabindex="-1"></a><span class="im">import</span> matplotlib.pyplot <span class="im">as</span> plt</span>
+<span id="cb21-3"><a href="#cb21-3" tabindex="-1"></a><span class="im">from</span> sklearn.metrics <span class="im">import</span> confusion_matrix, ConfusionMatrixDisplay</span>
+<span id="cb21-4"><a href="#cb21-4" tabindex="-1"></a></span>
+<span id="cb21-5"><a href="#cb21-5" tabindex="-1"></a><span class="co"># Threshold value for the energy score</span></span>
+<span id="cb21-6"><a href="#cb21-6" tabindex="-1"></a>upper_threshold <span class="op">=</span> best_f1_threshold  <span class="co"># Using the best F1 threshold from the previous calculation</span></span>
+<span id="cb21-7"><a href="#cb21-7" tabindex="-1"></a></span>
+<span id="cb21-8"><a href="#cb21-8" tabindex="-1"></a><span class="co"># Classifying OOD examples based on energy scores</span></span>
+<span id="cb21-9"><a href="#cb21-9" tabindex="-1"></a>ood_classifications <span class="op">=</span> np.where(ood_energy_scores <span class="op">&gt;=</span> upper_threshold, <span class="op">-</span><span class="dv">1</span>,  <span class="co"># classified as OOD</span></span>
+<span id="cb21-10"><a href="#cb21-10" tabindex="-1"></a>                               np.where(ood_energy_scores <span class="op">&lt;</span> upper_threshold, <span class="dv">0</span>, <span class="op">-</span><span class="dv">1</span>))  <span class="co"># classified as ID</span></span>
+<span id="cb21-11"><a href="#cb21-11" tabindex="-1"></a></span>
+<span id="cb21-12"><a href="#cb21-12" tabindex="-1"></a><span class="co"># Classifying ID examples based on energy scores</span></span>
+<span id="cb21-13"><a href="#cb21-13" tabindex="-1"></a>id_classifications <span class="op">=</span> np.where(id_energy_scores <span class="op">&gt;=</span> upper_threshold, <span class="op">-</span><span class="dv">1</span>,  <span class="co"># classified as OOD</span></span>
+<span id="cb21-14"><a href="#cb21-14" tabindex="-1"></a>                              np.where(id_energy_scores <span class="op">&lt;</span> upper_threshold, id_true_labels, <span class="op">-</span><span class="dv">1</span>))  <span class="co"># classified as ID</span></span>
+<span id="cb21-15"><a href="#cb21-15" tabindex="-1"></a></span>
+<span id="cb21-16"><a href="#cb21-16" tabindex="-1"></a><span class="co"># Combine OOD and ID classifications and true labels</span></span>
+<span id="cb21-17"><a href="#cb21-17" tabindex="-1"></a>all_predictions <span class="op">=</span> np.concatenate([ood_classifications, id_classifications])</span>
+<span id="cb21-18"><a href="#cb21-18" tabindex="-1"></a>all_true_labels <span class="op">=</span> np.concatenate([ood_true_labels, id_true_labels])</span>
+<span id="cb21-19"><a href="#cb21-19" tabindex="-1"></a></span>
+<span id="cb21-20"><a href="#cb21-20" tabindex="-1"></a><span class="co"># Confusion matrix</span></span>
+<span id="cb21-21"><a href="#cb21-21" tabindex="-1"></a>cm <span class="op">=</span> confusion_matrix(all_true_labels, all_predictions, labels<span class="op">=</span>[<span class="dv">0</span>, <span class="dv">1</span>, <span class="op">-</span><span class="dv">1</span>])</span>
+<span id="cb21-22"><a href="#cb21-22" tabindex="-1"></a></span>
+<span id="cb21-23"><a href="#cb21-23" tabindex="-1"></a><span class="co"># Plotting the confusion matrix</span></span>
+<span id="cb21-24"><a href="#cb21-24" tabindex="-1"></a>disp <span class="op">=</span> ConfusionMatrixDisplay(confusion_matrix<span class="op">=</span>cm, display_labels<span class="op">=</span>[<span class="st">"Shirt"</span>, <span class="st">"Pants"</span>, <span class="st">"OOD"</span>])</span>
+<span id="cb21-25"><a href="#cb21-25" tabindex="-1"></a>disp.plot(cmap<span class="op">=</span>plt.cm.Blues)</span>
+<span id="cb21-26"><a href="#cb21-26" tabindex="-1"></a>plt.title(<span class="st">'Confusion Matrix for OOD and ID Classification (Energy-Based)'</span>)</span>
+<span id="cb21-27"><a href="#cb21-27" tabindex="-1"></a>plt.show()</span></code></pre>
+</div>
+</div>
+</div>
+</div>
+<div class="section level1">
+<h1 id="conclusion">Conclusion<a class="anchor" aria-label="anchor" href="#conclusion"></a>
+</h1>
+<div class="codewrapper sourceCode" id="cb22">
 <h3 class="code-label">PYTHON<i aria-hidden="true" data-feather="chevron-left"></i><i aria-hidden="true" data-feather="chevron-right"></i>
 </h3>
 <pre class="sourceCode python" tabindex="0"><code class="sourceCode python"></code></pre>
 </div>
+<div class="section level2">
+<h2 id="references-and-supplemental-resources">References and supplemental resources<a class="anchor" aria-label="anchor" href="#references-and-supplemental-resources"></a>
+</h2>
+<ul>
+<li><a href="https://www.youtube.com/watch?v=hgLC9_9ZCJI" class="external-link uri">https://www.youtube.com/watch?v=hgLC9_9ZCJI</a></li>
+<li>Generalized Out-of-Distribution Detection: A Survey: <a href="https://arxiv.org/abs/2110.11334" class="external-link uri">https://arxiv.org/abs/2110.11334</a> # Glossary</li>
+<li>ID/OOD: In-distribution, out-of-distribution. Generally, the OOD
+instances can be defined as instances (x, y) sampled from an underlying
+distribution other than the training distribution P(Xtrain, Ytrain),
+where Xtrain and Ytrain are the training corpus and training label set,
+respectively.</li>
+<li>OOD instances with semantic shift: OOD instances with semantic shift
+refer to instances that do not belong to y_train. More specifically,
+instances with semantic shift may come from unknown categories or
+irrelevant tasks.</li>
+<li>OOD instances with covariate shift: OOD instances with non-semantic
+shift refer to the instances that belong to y_train but are sampled from
+a distribution other than x_train, e.g., a different
+domain/corpus/location.</li>
+<li>Closed-world assumption: an assumption that the training and test
+data are sampled from the same distribution. However, training data can
+rarely capture the entire distribution. In real-world scenarios,
+out-of-distribution (OOD) instances, which come from categories that are
+not known to the model, can often be present in inference phases.</li>
+<li>Inference-time OOD: After training, use some kind of scoring
+function to determine if test inputs are OOD or not.</li>
+<li>Output-based OOD: Output-based OOD detection methods leverage the
+model’s output distribution to identify OOD instances. These methods
+typically involve analyzing the softmax scores, confidence scores, or
+other output statistics to detect anomalies.</li>
+</ul>
 <!--
 Place links that you need to refer to multiple times across pages here. Delete
 any links that you are not going to use.
  -->
+</div>
 </div></section><section id="aio-7b-OOD-detection-distance-based"><p>Content from <a href="7b-OOD-detection-distance-based.html">OOD Detection: Distance-Based and Contrastive Learning</a></p>
 <hr>
 <p>Last updated on 2024-07-31 |
diff --git a/images.html b/images.html
index 71cbbbc6..b4abc341 100644
--- a/images.html
+++ b/images.html
@@ -537,7 +537,7 @@ <h3 id="5a-explainable-AI-method-overview-figure-6">Figure 6</h3>
 <hr class="half-width"></section><section id="5c-probes"><h2 class="section-heading"><a href="5c-probes.html">Explainability methods: linear probe</a></h2>
 <hr class="half-width"></section><section id="5d-gradcam"><h2 class="section-heading"><a href="5d-gradcam.html">Explainability methods: GradCAM</a></h2>
 <hr class="half-width"></section><section id="6-confidence-intervals"><h2 class="section-heading"><a href="6-confidence-intervals.html">Estimating model uncertainty</a></h2>
-<hr class="half-width"></section><section id="7a-OOD-detection-output-based"><h2 class="section-heading"><a href="7a-OOD-detection-output-based.html">OOD Detection: Overview, Output-Based MethodsIntroduction to Out-of-Distribution (OOD) Data<a class="anchor" aria-label="anchor" href="#introduction-to-out-of-distribution-ood-data"></a>Example 1: Softmax scores<a class="anchor" aria-label="anchor" href="#example-1-softmax-scores"></a>Example 2: Energy-Based OOD Detection<a class="anchor" aria-label="anchor" href="#example-2-energy-based-ood-detection"></a></a></h2>
+<hr class="half-width"></section><section id="7a-OOD-detection-output-based"><h2 class="section-heading"><a href="7a-OOD-detection-output-based.html">OOD Detection: Overview, Output-Based MethodsIntroduction to Out-of-Distribution (OOD) Data<a class="anchor" aria-label="anchor" href="#introduction-to-out-of-distribution-ood-data"></a>Example 1: Softmax scores<a class="anchor" aria-label="anchor" href="#example-1-softmax-scores"></a>Example 2: Energy-Based OOD Detection<a class="anchor" aria-label="anchor" href="#example-2-energy-based-ood-detection"></a>Conclusion<a class="anchor" aria-label="anchor" href="#conclusion"></a></a></h2>
 <hr class="half-width"></section><section id="7b-OOD-detection-distance-based"><h2 class="section-heading"><a href="7b-OOD-detection-distance-based.html">OOD Detection: Distance-Based and Contrastive LearningExample 3: Distance-Based Methods<a class="anchor" aria-label="anchor" href="#example-3-distance-based-methods"></a>Limitations of Threshold-Based OOD Detection Methods<a class="anchor" aria-label="anchor" href="#limitations-of-threshold-based-ood-detection-methods"></a></a></h2>
 <hr class="half-width"></section><section id="7c-OOD-detection-algo-design"><h2 class="section-heading"><a href="7c-OOD-detection-algo-design.html">OOD Detection: Training-Time RegularizationTraining-time regularization for OOD detection<a class="anchor" aria-label="anchor" href="#training-time-regularization-for-ood-detection"></a></a></h2>
 <hr class="half-width"></section><section id="8-releasing-a-model"><h2 class="section-heading"><a href="8-releasing-a-model.html">Documenting and releasing a model</a></h2>
diff --git a/instructor/1-preparing-to-train.html b/instructor/1-preparing-to-train.html
index 837f0587..c7bf5d6c 100644
--- a/instructor/1-preparing-to-train.html
+++ b/instructor/1-preparing-to-train.html
@@ -474,7 +474,7 @@ <h3 class="callout-title">Challenge<a class="anchor" aria-label="anchor" href="#
 <button class="accordion-button solution-button collapsed" type="button" data-bs-toggle="collapse" data-bs-target="#collapseSolution1" aria-expanded="false" aria-controls="collapseSolution1">
   <h4 class="accordion-header" id="headingSolution1"> Show me the solution </h4>
 </button>
-<div id="collapseSolution1" class="accordion-collapse collapse" aria-labelledby="headingSolution1" data-bs-parent="#accordionSolution1">
+<div id="collapseSolution1" class="accordion-collapse collapse" data-bs-parent="#accordionSolution1" aria-labelledby="headingSolution1">
 <div class="accordion-body">
 <p>A summary of the principles is listed below:</p>
 <ul><li>Social and clinical value: Does the social or clinical value of
diff --git a/instructor/5a-explainable-AI-method-overview.html b/instructor/5a-explainable-AI-method-overview.html
index 8920633b..1cf04663 100644
--- a/instructor/5a-explainable-AI-method-overview.html
+++ b/instructor/5a-explainable-AI-method-overview.html
@@ -693,7 +693,7 @@ <h3 class="callout-title">Classifying explanation techniques<a class="anchor" ar
 <button class="accordion-button solution-button collapsed" type="button" data-bs-toggle="collapse" data-bs-target="#collapseSolution1" aria-expanded="false" aria-controls="collapseSolution1">
   <h4 class="accordion-header" id="headingSolution1"> Show me the solution </h4>
 </button>
-<div id="collapseSolution1" class="accordion-collapse collapse" data-bs-parent="#accordionSolution1" aria-labelledby="headingSolution1">
+<div id="collapseSolution1" class="accordion-collapse collapse" aria-labelledby="headingSolution1" data-bs-parent="#accordionSolution1">
 <div class="accordion-body">
 <table class="table"><colgroup><col width="54%"><col width="21%"><col width="10%"><col width="13%"></colgroup><thead><tr class="header"><th>Approach</th>
 <th>Post Hoc or Inherently Interpretable?</th>
diff --git a/instructor/7a-OOD-detection-output-based.html b/instructor/7a-OOD-detection-output-based.html
index 92b8bd68..df470da2 100644
--- a/instructor/7a-OOD-detection-output-based.html
+++ b/instructor/7a-OOD-detection-output-based.html
@@ -414,8 +414,8 @@ <h2 class="card-header">Overview</h2>
 <h3 class="card-title">Questions</h3>
 <ul><li>What are out-of-distribution (OOD) data and why is detecting them
 important in machine learning models?</li>
-<li>How do output-based methods like softmax, energy-based, and
-distance-based methods work for OOD detection?</li>
+<li>How do output-based methods like softmax and energy-based methods
+work for OOD detection?</li>
 <li>What are the limitations of output-based OOD detection methods?</li>
 </ul></div>
 </div>
@@ -990,7 +990,7 @@ <h3 class="code-label">PYTHON<i aria-hidden="true" data-feather="chevron-left"><
 <span id="cb14-29"><a href="#cb14-29" tabindex="-1"></a>  all_true_labels <span class="op">=</span> np.concatenate([<span class="op">-</span><span class="dv">1</span> <span class="op">*</span> np.ones(ood_classifications.shape), train_labels])</span>
 <span id="cb14-30"><a href="#cb14-30" tabindex="-1"></a></span>
 <span id="cb14-31"><a href="#cb14-31" tabindex="-1"></a>  <span class="co"># Evaluate metrics</span></span>
-<span id="cb14-32"><a href="#cb14-32" tabindex="-1"></a>  precision, recall, f1, _ <span class="op">=</span> precision_recall_fscore_support(all_true_labels, all_predictions, labels<span class="op">=</span>[<span class="dv">0</span>, <span class="dv">1</span>], average<span class="op">=</span><span class="st">'macro'</span>) <span class="co"># discuss macro vs micro .</span></span>
+<span id="cb14-32"><a href="#cb14-32" tabindex="-1"></a>  precision, recall, f1, _ <span class="op">=</span> precision_recall_fscore_support(all_true_labels, all_predictions, labels<span class="op">=</span>[<span class="dv">0</span>, <span class="dv">1</span>, <span class="op">-</span><span class="dv">1</span>], average<span class="op">=</span><span class="st">'macro'</span>) <span class="co"># discuss macro vs micro .</span></span>
 <span id="cb14-33"><a href="#cb14-33" tabindex="-1"></a>  accuracy <span class="op">=</span> accuracy_score(all_true_labels, all_predictions)</span>
 <span id="cb14-34"><a href="#cb14-34" tabindex="-1"></a></span>
 <span id="cb14-35"><a href="#cb14-35" tabindex="-1"></a>  accuracies.append(accuracy)</span>
@@ -1029,13 +1029,51 @@ <h3 class="code-label">PYTHON<i aria-hidden="true" data-feather="chevron-left"><
 <span id="cb14-68"><a href="#cb14-68" tabindex="-1"></a>plt.legend()</span>
 <span id="cb14-69"><a href="#cb14-69" tabindex="-1"></a>plt.show()</span></code></pre>
 </div>
+<div class="codewrapper sourceCode" id="cb15">
+<h3 class="code-label">PYTHON<i aria-hidden="true" data-feather="chevron-left"></i><i aria-hidden="true" data-feather="chevron-right"></i>
+</h3>
+<pre class="sourceCode python" tabindex="0"><code class="sourceCode python"><span id="cb15-1"><a href="#cb15-1" tabindex="-1"></a><span class="im">import</span> numpy <span class="im">as</span> np</span>
+<span id="cb15-2"><a href="#cb15-2" tabindex="-1"></a><span class="im">import</span> matplotlib.pyplot <span class="im">as</span> plt</span>
+<span id="cb15-3"><a href="#cb15-3" tabindex="-1"></a><span class="im">from</span> sklearn.metrics <span class="im">import</span> confusion_matrix, ConfusionMatrixDisplay</span>
+<span id="cb15-4"><a href="#cb15-4" tabindex="-1"></a></span>
+<span id="cb15-5"><a href="#cb15-5" tabindex="-1"></a><span class="co"># Assuming ood_probs, id_probs, and train_labels are defined</span></span>
+<span id="cb15-6"><a href="#cb15-6" tabindex="-1"></a><span class="co"># Threshold values</span></span>
+<span id="cb15-7"><a href="#cb15-7" tabindex="-1"></a>upper_threshold <span class="op">=</span> best_f1_threshold</span>
+<span id="cb15-8"><a href="#cb15-8" tabindex="-1"></a></span>
+<span id="cb15-9"><a href="#cb15-9" tabindex="-1"></a><span class="co"># Classifying OOD examples (sandals)</span></span>
+<span id="cb15-10"><a href="#cb15-10" tabindex="-1"></a>ood_classifications <span class="op">=</span> np.where(ood_probs[:, <span class="dv">1</span>] <span class="op">&gt;=</span> upper_threshold, <span class="dv">1</span>,  <span class="co"># classified as pants</span></span>
+<span id="cb15-11"><a href="#cb15-11" tabindex="-1"></a>                               np.where(ood_probs[:, <span class="dv">0</span>] <span class="op">&gt;=</span> upper_threshold, <span class="dv">0</span>,  <span class="co"># classified as shirts</span></span>
+<span id="cb15-12"><a href="#cb15-12" tabindex="-1"></a>                                        <span class="op">-</span><span class="dv">1</span>))  <span class="co"># classified as OOD</span></span>
+<span id="cb15-13"><a href="#cb15-13" tabindex="-1"></a>ood_classifications</span>
+<span id="cb15-14"><a href="#cb15-14" tabindex="-1"></a></span>
+<span id="cb15-15"><a href="#cb15-15" tabindex="-1"></a>id_probs</span>
+<span id="cb15-16"><a href="#cb15-16" tabindex="-1"></a><span class="co"># Classifying ID examples (T-shirts and pants)</span></span>
+<span id="cb15-17"><a href="#cb15-17" tabindex="-1"></a>id_classifications <span class="op">=</span> np.where(id_probs[:, <span class="dv">1</span>] <span class="op">&gt;=</span> upper_threshold, <span class="dv">1</span>,  <span class="co"># classified as pants</span></span>
+<span id="cb15-18"><a href="#cb15-18" tabindex="-1"></a>                              np.where(id_probs[:, <span class="dv">0</span>] <span class="op">&gt;=</span> upper_threshold, <span class="dv">0</span>,  <span class="co"># classified as shirts</span></span>
+<span id="cb15-19"><a href="#cb15-19" tabindex="-1"></a>                                       <span class="op">-</span><span class="dv">1</span>))  <span class="co"># classified as OOD</span></span>
+<span id="cb15-20"><a href="#cb15-20" tabindex="-1"></a></span>
+<span id="cb15-21"><a href="#cb15-21" tabindex="-1"></a>id_classifications</span>
+<span id="cb15-22"><a href="#cb15-22" tabindex="-1"></a></span>
+<span id="cb15-23"><a href="#cb15-23" tabindex="-1"></a><span class="co"># Combine OOD and ID classifications and true labels</span></span>
+<span id="cb15-24"><a href="#cb15-24" tabindex="-1"></a>all_predictions <span class="op">=</span> np.concatenate([ood_classifications, id_classifications])</span>
+<span id="cb15-25"><a href="#cb15-25" tabindex="-1"></a>all_true_labels <span class="op">=</span> np.concatenate([<span class="op">-</span><span class="dv">1</span> <span class="op">*</span> np.ones(ood_classifications.shape), train_labels])</span>
+<span id="cb15-26"><a href="#cb15-26" tabindex="-1"></a></span>
+<span id="cb15-27"><a href="#cb15-27" tabindex="-1"></a><span class="co"># Confusion matrix</span></span>
+<span id="cb15-28"><a href="#cb15-28" tabindex="-1"></a>cm <span class="op">=</span> confusion_matrix(all_true_labels, all_predictions, labels<span class="op">=</span>[<span class="dv">0</span>, <span class="dv">1</span>, <span class="op">-</span><span class="dv">1</span>])</span>
+<span id="cb15-29"><a href="#cb15-29" tabindex="-1"></a></span>
+<span id="cb15-30"><a href="#cb15-30" tabindex="-1"></a><span class="co"># Plotting the confusion matrix</span></span>
+<span id="cb15-31"><a href="#cb15-31" tabindex="-1"></a>disp <span class="op">=</span> ConfusionMatrixDisplay(confusion_matrix<span class="op">=</span>cm, display_labels<span class="op">=</span>[<span class="st">"Shirt"</span>, <span class="st">"Pants"</span>, <span class="st">"OOD"</span>])</span>
+<span id="cb15-32"><a href="#cb15-32" tabindex="-1"></a>disp.plot(cmap<span class="op">=</span>plt.cm.Blues)</span>
+<span id="cb15-33"><a href="#cb15-33" tabindex="-1"></a>plt.title(<span class="st">'Confusion Matrix for OOD and ID Classification'</span>)</span>
+<span id="cb15-34"><a href="#cb15-34" tabindex="-1"></a>plt.show()</span></code></pre>
+</div>
 </div>
 </div>
 </div>
 <div class="section level1">
 <h1 id="example-2-energy-based-ood-detection">Example 2: Energy-Based OOD Detection<a class="anchor" aria-label="anchor" href="#example-2-energy-based-ood-detection"></a></h1>
-<p>Liu et al., Energy-based Out-of-distribution Detection, NeurIPS
-2020</p>
+<p>Liu et al., Energy-based Out-of-distribution Detection, NeurIPS 2020;
+<a href="https://arxiv.org/pdf/2010.03759" class="external-link uri">https://arxiv.org/pdf/2010.03759</a></p>
 <ul><li><p>E(x, y) = energy value</p></li>
 <li><p>if x and y are “compatitble”, lower energy</p></li>
 <li>
@@ -1045,16 +1083,369 @@ <h1 id="example-2-energy-based-ood-detection">Example 2: Energy-Based OOD Detect
 </ul></li>
 <li><p>With energy scores, ID and OOD distributions become much more
 separable</p></li>
-<li><p>Another “output-based” method like softmax # Conclusion</p></li>
-</ul><div class="codewrapper sourceCode" id="cb15">
+<li><p>Another “output-based” method like softmax</p></li>
+</ul><div class="section level2">
+<h2 id="pytorch-out-of-distribution-detection">PyTorch Out-of-Distribution Detection<a class="anchor" aria-label="anchor" href="#pytorch-out-of-distribution-detection"></a></h2>
+<p>There’s a Pytorch package for OOD detection! <a href="https://pytorch-ood.readthedocs.io/en/latest/info.html" class="external-link uri">https://pytorch-ood.readthedocs.io/en/latest/info.html</a></p>
+<div class="codewrapper sourceCode" id="cb16">
+<h3 class="code-label">PYTHON<i aria-hidden="true" data-feather="chevron-left"></i><i aria-hidden="true" data-feather="chevron-right"></i>
+</h3>
+<pre class="sourceCode python" tabindex="0"><code class="sourceCode python"><span id="cb16-1"><a href="#cb16-1" tabindex="-1"></a><span class="op">!</span>pip install pytorch<span class="op">-</span>ood</span></code></pre>
+</div>
+<div class="section level3">
+<h3 id="energy-based-is-designed-to-work-with-neural-nets-unpack-this-">Energy-based is designed to work with neural nets… unpack this.<a class="anchor" aria-label="anchor" href="#energy-based-is-designed-to-work-with-neural-nets-unpack-this-"></a></h3>
+<p>Let’s train a simple CNN model on the FashionMNIST dataset.</p>
+<div class="codewrapper sourceCode" id="cb17">
+<h3 class="code-label">PYTHON<i aria-hidden="true" data-feather="chevron-left"></i><i aria-hidden="true" data-feather="chevron-right"></i>
+</h3>
+<pre class="sourceCode python" tabindex="0"><code class="sourceCode python"><span id="cb17-1"><a href="#cb17-1" tabindex="-1"></a><span class="im">import</span> torch</span>
+<span id="cb17-2"><a href="#cb17-2" tabindex="-1"></a><span class="im">import</span> torch.nn <span class="im">as</span> nn</span>
+<span id="cb17-3"><a href="#cb17-3" tabindex="-1"></a><span class="im">import</span> torch.optim <span class="im">as</span> optim</span>
+<span id="cb17-4"><a href="#cb17-4" tabindex="-1"></a><span class="im">import</span> torchvision.transforms <span class="im">as</span> transforms</span>
+<span id="cb17-5"><a href="#cb17-5" tabindex="-1"></a><span class="im">from</span> keras.datasets <span class="im">import</span> fashion_mnist</span>
+<span id="cb17-6"><a href="#cb17-6" tabindex="-1"></a><span class="im">import</span> torch.nn.functional <span class="im">as</span> F</span>
+<span id="cb17-7"><a href="#cb17-7" tabindex="-1"></a><span class="im">import</span> matplotlib.pyplot <span class="im">as</span> plt</span>
+<span id="cb17-8"><a href="#cb17-8" tabindex="-1"></a><span class="im">import</span> numpy <span class="im">as</span> np</span>
+<span id="cb17-9"><a href="#cb17-9" tabindex="-1"></a></span>
+<span id="cb17-10"><a href="#cb17-10" tabindex="-1"></a><span class="co"># Load Fashion MNIST dataset</span></span>
+<span id="cb17-11"><a href="#cb17-11" tabindex="-1"></a>(train_images, train_labels), (test_images, test_labels) <span class="op">=</span> fashion_mnist.load_data()</span>
+<span id="cb17-12"><a href="#cb17-12" tabindex="-1"></a></span>
+<span id="cb17-13"><a href="#cb17-13" tabindex="-1"></a><span class="co"># Define classes for simplicity</span></span>
+<span id="cb17-14"><a href="#cb17-14" tabindex="-1"></a>class_names <span class="op">=</span> [<span class="st">'T-shirt/top'</span>, <span class="st">'Trouser'</span>, <span class="st">'Pullover'</span>, <span class="st">'Dress'</span>, <span class="st">'Coat'</span>,</span>
+<span id="cb17-15"><a href="#cb17-15" tabindex="-1"></a>               <span class="st">'Sandal'</span>, <span class="st">'Shirt'</span>, <span class="st">'Sneaker'</span>, <span class="st">'Bag'</span>, <span class="st">'Ankle boot'</span>]</span>
+<span id="cb17-16"><a href="#cb17-16" tabindex="-1"></a></span>
+<span id="cb17-17"><a href="#cb17-17" tabindex="-1"></a><span class="co"># Prepare OOD data - Sandals (5)</span></span>
+<span id="cb17-18"><a href="#cb17-18" tabindex="-1"></a>ood_data <span class="op">=</span> test_images[test_labels <span class="op">==</span> <span class="dv">5</span>]</span>
+<span id="cb17-19"><a href="#cb17-19" tabindex="-1"></a>ood_labels <span class="op">=</span> test_labels[test_labels <span class="op">==</span> <span class="dv">5</span>]</span>
+<span id="cb17-20"><a href="#cb17-20" tabindex="-1"></a><span class="bu">print</span>(<span class="ss">f'ood_data.shape=</span><span class="sc">{</span>ood_data<span class="sc">.</span>shape<span class="sc">}</span><span class="ss">'</span>)</span>
+<span id="cb17-21"><a href="#cb17-21" tabindex="-1"></a></span>
+<span id="cb17-22"><a href="#cb17-22" tabindex="-1"></a><span class="co"># Filter data for T-shirts (0) and Trousers (1) as in-distribution</span></span>
+<span id="cb17-23"><a href="#cb17-23" tabindex="-1"></a>train_filter <span class="op">=</span> np.isin(train_labels, [<span class="dv">0</span>, <span class="dv">1</span>])</span>
+<span id="cb17-24"><a href="#cb17-24" tabindex="-1"></a>test_filter <span class="op">=</span> np.isin(test_labels, [<span class="dv">0</span>, <span class="dv">1</span>])</span>
+<span id="cb17-25"><a href="#cb17-25" tabindex="-1"></a></span>
+<span id="cb17-26"><a href="#cb17-26" tabindex="-1"></a>train_data <span class="op">=</span> train_images[train_filter]</span>
+<span id="cb17-27"><a href="#cb17-27" tabindex="-1"></a>train_labels <span class="op">=</span> train_labels[train_filter]</span>
+<span id="cb17-28"><a href="#cb17-28" tabindex="-1"></a><span class="bu">print</span>(<span class="ss">f'train_data.shape=</span><span class="sc">{</span>train_data<span class="sc">.</span>shape<span class="sc">}</span><span class="ss">'</span>)</span>
+<span id="cb17-29"><a href="#cb17-29" tabindex="-1"></a></span>
+<span id="cb17-30"><a href="#cb17-30" tabindex="-1"></a>test_data <span class="op">=</span> test_images[test_filter]</span>
+<span id="cb17-31"><a href="#cb17-31" tabindex="-1"></a>test_labels <span class="op">=</span> test_labels[test_filter]</span>
+<span id="cb17-32"><a href="#cb17-32" tabindex="-1"></a><span class="bu">print</span>(<span class="ss">f'test_data.shape=</span><span class="sc">{</span>test_data<span class="sc">.</span>shape<span class="sc">}</span><span class="ss">'</span>)</span>
+<span id="cb17-33"><a href="#cb17-33" tabindex="-1"></a></span>
+<span id="cb17-34"><a href="#cb17-34" tabindex="-1"></a><span class="co"># Transform to Tensor and normalize</span></span>
+<span id="cb17-35"><a href="#cb17-35" tabindex="-1"></a>transform <span class="op">=</span> transforms.Compose([</span>
+<span id="cb17-36"><a href="#cb17-36" tabindex="-1"></a>    transforms.ToTensor(),</span>
+<span id="cb17-37"><a href="#cb17-37" tabindex="-1"></a>    transforms.Normalize((<span class="fl">0.5</span>,), (<span class="fl">0.5</span>,))</span>
+<span id="cb17-38"><a href="#cb17-38" tabindex="-1"></a>])</span>
+<span id="cb17-39"><a href="#cb17-39" tabindex="-1"></a></span>
+<span id="cb17-40"><a href="#cb17-40" tabindex="-1"></a><span class="co"># Convert to PyTorch tensors and normalize</span></span>
+<span id="cb17-41"><a href="#cb17-41" tabindex="-1"></a>train_data_tensor <span class="op">=</span> torch.tensor(train_data, dtype<span class="op">=</span>torch.float32).unsqueeze(<span class="dv">1</span>) <span class="op">/</span> <span class="fl">255.0</span></span>
+<span id="cb17-42"><a href="#cb17-42" tabindex="-1"></a>test_data_tensor <span class="op">=</span> torch.tensor(test_data, dtype<span class="op">=</span>torch.float32).unsqueeze(<span class="dv">1</span>) <span class="op">/</span> <span class="fl">255.0</span></span>
+<span id="cb17-43"><a href="#cb17-43" tabindex="-1"></a>ood_data_tensor <span class="op">=</span> torch.tensor(ood_data, dtype<span class="op">=</span>torch.float32).unsqueeze(<span class="dv">1</span>) <span class="op">/</span> <span class="fl">255.0</span></span>
+<span id="cb17-44"><a href="#cb17-44" tabindex="-1"></a></span>
+<span id="cb17-45"><a href="#cb17-45" tabindex="-1"></a>train_labels_tensor <span class="op">=</span> torch.tensor(train_labels, dtype<span class="op">=</span>torch.<span class="bu">long</span>)</span>
+<span id="cb17-46"><a href="#cb17-46" tabindex="-1"></a>test_labels_tensor <span class="op">=</span> torch.tensor(test_labels, dtype<span class="op">=</span>torch.<span class="bu">long</span>)</span>
+<span id="cb17-47"><a href="#cb17-47" tabindex="-1"></a></span>
+<span id="cb17-48"><a href="#cb17-48" tabindex="-1"></a>train_dataset <span class="op">=</span> torch.utils.data.TensorDataset(train_data_tensor, train_labels_tensor)</span>
+<span id="cb17-49"><a href="#cb17-49" tabindex="-1"></a>test_dataset <span class="op">=</span> torch.utils.data.TensorDataset(test_data_tensor, test_labels_tensor)</span>
+<span id="cb17-50"><a href="#cb17-50" tabindex="-1"></a>ood_dataset <span class="op">=</span> torch.utils.data.TensorDataset(ood_data_tensor, torch.zeros(ood_data_tensor.shape[<span class="dv">0</span>], dtype<span class="op">=</span>torch.<span class="bu">long</span>))</span>
+<span id="cb17-51"><a href="#cb17-51" tabindex="-1"></a></span>
+<span id="cb17-52"><a href="#cb17-52" tabindex="-1"></a>train_loader <span class="op">=</span> torch.utils.data.DataLoader(train_dataset, batch_size<span class="op">=</span><span class="dv">64</span>, shuffle<span class="op">=</span><span class="va">True</span>)</span>
+<span id="cb17-53"><a href="#cb17-53" tabindex="-1"></a>test_loader <span class="op">=</span> torch.utils.data.DataLoader(test_dataset, batch_size<span class="op">=</span><span class="dv">64</span>, shuffle<span class="op">=</span><span class="va">False</span>)</span>
+<span id="cb17-54"><a href="#cb17-54" tabindex="-1"></a>ood_loader <span class="op">=</span> torch.utils.data.DataLoader(ood_dataset, batch_size<span class="op">=</span><span class="dv">64</span>, shuffle<span class="op">=</span><span class="va">False</span>)</span>
+<span id="cb17-55"><a href="#cb17-55" tabindex="-1"></a></span>
+<span id="cb17-56"><a href="#cb17-56" tabindex="-1"></a><span class="co"># Define a simple CNN model</span></span>
+<span id="cb17-57"><a href="#cb17-57" tabindex="-1"></a><span class="kw">class</span> SimpleCNN(nn.Module):</span>
+<span id="cb17-58"><a href="#cb17-58" tabindex="-1"></a>    <span class="kw">def</span> <span class="fu">__init__</span>(<span class="va">self</span>):</span>
+<span id="cb17-59"><a href="#cb17-59" tabindex="-1"></a>        <span class="bu">super</span>(SimpleCNN, <span class="va">self</span>).<span class="fu">__init__</span>()</span>
+<span id="cb17-60"><a href="#cb17-60" tabindex="-1"></a>        <span class="va">self</span>.conv1 <span class="op">=</span> nn.Conv2d(<span class="dv">1</span>, <span class="dv">32</span>, kernel_size<span class="op">=</span><span class="dv">3</span>)</span>
+<span id="cb17-61"><a href="#cb17-61" tabindex="-1"></a>        <span class="va">self</span>.conv2 <span class="op">=</span> nn.Conv2d(<span class="dv">32</span>, <span class="dv">64</span>, kernel_size<span class="op">=</span><span class="dv">3</span>)</span>
+<span id="cb17-62"><a href="#cb17-62" tabindex="-1"></a>        <span class="va">self</span>.fc1 <span class="op">=</span> nn.Linear(<span class="dv">64</span><span class="op">*</span><span class="dv">5</span><span class="op">*</span><span class="dv">5</span>, <span class="dv">128</span>)  <span class="co"># Updated this line</span></span>
+<span id="cb17-63"><a href="#cb17-63" tabindex="-1"></a>        <span class="va">self</span>.fc2 <span class="op">=</span> nn.Linear(<span class="dv">128</span>, <span class="dv">2</span>)</span>
+<span id="cb17-64"><a href="#cb17-64" tabindex="-1"></a></span>
+<span id="cb17-65"><a href="#cb17-65" tabindex="-1"></a>    <span class="kw">def</span> forward(<span class="va">self</span>, x):</span>
+<span id="cb17-66"><a href="#cb17-66" tabindex="-1"></a>        x <span class="op">=</span> F.relu(F.max_pool2d(<span class="va">self</span>.conv1(x), <span class="dv">2</span>))</span>
+<span id="cb17-67"><a href="#cb17-67" tabindex="-1"></a>        x <span class="op">=</span> F.relu(F.max_pool2d(<span class="va">self</span>.conv2(x), <span class="dv">2</span>))</span>
+<span id="cb17-68"><a href="#cb17-68" tabindex="-1"></a>        x <span class="op">=</span> x.view(<span class="op">-</span><span class="dv">1</span>, <span class="dv">64</span><span class="op">*</span><span class="dv">5</span><span class="op">*</span><span class="dv">5</span>)  <span class="co"># Updated this line</span></span>
+<span id="cb17-69"><a href="#cb17-69" tabindex="-1"></a>        x <span class="op">=</span> F.relu(<span class="va">self</span>.fc1(x))</span>
+<span id="cb17-70"><a href="#cb17-70" tabindex="-1"></a>        x <span class="op">=</span> <span class="va">self</span>.fc2(x)</span>
+<span id="cb17-71"><a href="#cb17-71" tabindex="-1"></a>        <span class="cf">return</span> x</span>
+<span id="cb17-72"><a href="#cb17-72" tabindex="-1"></a></span>
+<span id="cb17-73"><a href="#cb17-73" tabindex="-1"></a>device <span class="op">=</span> torch.device(<span class="st">'cuda'</span> <span class="cf">if</span> torch.cuda.is_available() <span class="cf">else</span> <span class="st">'cpu'</span>)</span>
+<span id="cb17-74"><a href="#cb17-74" tabindex="-1"></a>model <span class="op">=</span> SimpleCNN().to(device)</span>
+<span id="cb17-75"><a href="#cb17-75" tabindex="-1"></a>criterion <span class="op">=</span> nn.CrossEntropyLoss()</span>
+<span id="cb17-76"><a href="#cb17-76" tabindex="-1"></a>optimizer <span class="op">=</span> optim.Adam(model.parameters(), lr<span class="op">=</span><span class="fl">0.001</span>)</span>
+<span id="cb17-77"><a href="#cb17-77" tabindex="-1"></a></span>
+<span id="cb17-78"><a href="#cb17-78" tabindex="-1"></a><span class="kw">def</span> train_model(model, train_loader, criterion, optimizer, epochs<span class="op">=</span><span class="dv">5</span>):</span>
+<span id="cb17-79"><a href="#cb17-79" tabindex="-1"></a>    model.train()</span>
+<span id="cb17-80"><a href="#cb17-80" tabindex="-1"></a>    <span class="cf">for</span> epoch <span class="kw">in</span> <span class="bu">range</span>(epochs):</span>
+<span id="cb17-81"><a href="#cb17-81" tabindex="-1"></a>        running_loss <span class="op">=</span> <span class="fl">0.0</span></span>
+<span id="cb17-82"><a href="#cb17-82" tabindex="-1"></a>        <span class="cf">for</span> inputs, labels <span class="kw">in</span> train_loader:</span>
+<span id="cb17-83"><a href="#cb17-83" tabindex="-1"></a>            inputs, labels <span class="op">=</span> inputs.to(device), labels.to(device)</span>
+<span id="cb17-84"><a href="#cb17-84" tabindex="-1"></a>            optimizer.zero_grad()</span>
+<span id="cb17-85"><a href="#cb17-85" tabindex="-1"></a>            outputs <span class="op">=</span> model(inputs)</span>
+<span id="cb17-86"><a href="#cb17-86" tabindex="-1"></a>            loss <span class="op">=</span> criterion(outputs, labels)</span>
+<span id="cb17-87"><a href="#cb17-87" tabindex="-1"></a>            loss.backward()</span>
+<span id="cb17-88"><a href="#cb17-88" tabindex="-1"></a>            optimizer.step()</span>
+<span id="cb17-89"><a href="#cb17-89" tabindex="-1"></a>            running_loss <span class="op">+=</span> loss.item()</span>
+<span id="cb17-90"><a href="#cb17-90" tabindex="-1"></a>        <span class="bu">print</span>(<span class="ss">f'Epoch </span><span class="sc">{</span>epoch<span class="op">+</span><span class="dv">1</span><span class="sc">}</span><span class="ss">, Loss: </span><span class="sc">{</span>running_loss<span class="op">/</span><span class="bu">len</span>(train_loader)<span class="sc">}</span><span class="ss">'</span>)</span>
+<span id="cb17-91"><a href="#cb17-91" tabindex="-1"></a></span>
+<span id="cb17-92"><a href="#cb17-92" tabindex="-1"></a>train_model(model, train_loader, criterion, optimizer)</span></code></pre>
+</div>
+<div class="codewrapper sourceCode" id="cb18">
+<h3 class="code-label">PYTHON<i aria-hidden="true" data-feather="chevron-left"></i><i aria-hidden="true" data-feather="chevron-right"></i>
+</h3>
+<pre class="sourceCode python" tabindex="0"><code class="sourceCode python"><span id="cb18-1"><a href="#cb18-1" tabindex="-1"></a><span class="im">from</span> sklearn.metrics <span class="im">import</span> confusion_matrix, ConfusionMatrixDisplay</span>
+<span id="cb18-2"><a href="#cb18-2" tabindex="-1"></a></span>
+<span id="cb18-3"><a href="#cb18-3" tabindex="-1"></a><span class="co"># Function to plot confusion matrix</span></span>
+<span id="cb18-4"><a href="#cb18-4" tabindex="-1"></a><span class="kw">def</span> plot_confusion_matrix(labels, predictions, title):</span>
+<span id="cb18-5"><a href="#cb18-5" tabindex="-1"></a>    cm <span class="op">=</span> confusion_matrix(labels, predictions, labels<span class="op">=</span>[<span class="dv">0</span>, <span class="dv">1</span>])</span>
+<span id="cb18-6"><a href="#cb18-6" tabindex="-1"></a>    disp <span class="op">=</span> ConfusionMatrixDisplay(confusion_matrix<span class="op">=</span>cm, display_labels<span class="op">=</span>[<span class="st">"T-shirt/top"</span>, <span class="st">"Trouser"</span>])</span>
+<span id="cb18-7"><a href="#cb18-7" tabindex="-1"></a>    disp.plot(cmap<span class="op">=</span>plt.cm.Blues)</span>
+<span id="cb18-8"><a href="#cb18-8" tabindex="-1"></a>    plt.title(title)</span>
+<span id="cb18-9"><a href="#cb18-9" tabindex="-1"></a>    plt.show()</span>
+<span id="cb18-10"><a href="#cb18-10" tabindex="-1"></a></span>
+<span id="cb18-11"><a href="#cb18-11" tabindex="-1"></a><span class="co"># Function to evaluate model on a dataset</span></span>
+<span id="cb18-12"><a href="#cb18-12" tabindex="-1"></a><span class="kw">def</span> evaluate_model(model, dataloader, device):</span>
+<span id="cb18-13"><a href="#cb18-13" tabindex="-1"></a>    model.<span class="bu">eval</span>()</span>
+<span id="cb18-14"><a href="#cb18-14" tabindex="-1"></a>    all_labels <span class="op">=</span> []</span>
+<span id="cb18-15"><a href="#cb18-15" tabindex="-1"></a>    all_predictions <span class="op">=</span> []</span>
+<span id="cb18-16"><a href="#cb18-16" tabindex="-1"></a>    <span class="cf">with</span> torch.no_grad():</span>
+<span id="cb18-17"><a href="#cb18-17" tabindex="-1"></a>        <span class="cf">for</span> inputs, labels <span class="kw">in</span> dataloader:</span>
+<span id="cb18-18"><a href="#cb18-18" tabindex="-1"></a>            inputs, labels <span class="op">=</span> inputs.to(device), labels.to(device)</span>
+<span id="cb18-19"><a href="#cb18-19" tabindex="-1"></a>            outputs <span class="op">=</span> model(inputs)</span>
+<span id="cb18-20"><a href="#cb18-20" tabindex="-1"></a>            _, preds <span class="op">=</span> torch.<span class="bu">max</span>(outputs, <span class="dv">1</span>)</span>
+<span id="cb18-21"><a href="#cb18-21" tabindex="-1"></a>            all_labels.extend(labels.cpu().numpy())</span>
+<span id="cb18-22"><a href="#cb18-22" tabindex="-1"></a>            all_predictions.extend(preds.cpu().numpy())</span>
+<span id="cb18-23"><a href="#cb18-23" tabindex="-1"></a>    <span class="cf">return</span> np.array(all_labels), np.array(all_predictions)</span>
+<span id="cb18-24"><a href="#cb18-24" tabindex="-1"></a></span>
+<span id="cb18-25"><a href="#cb18-25" tabindex="-1"></a><span class="co"># Evaluate on train data</span></span>
+<span id="cb18-26"><a href="#cb18-26" tabindex="-1"></a>train_labels, train_predictions <span class="op">=</span> evaluate_model(model, train_loader, device)</span>
+<span id="cb18-27"><a href="#cb18-27" tabindex="-1"></a>plot_confusion_matrix(train_labels, train_predictions, <span class="st">"Confusion Matrix for Train Data"</span>)</span>
+<span id="cb18-28"><a href="#cb18-28" tabindex="-1"></a></span>
+<span id="cb18-29"><a href="#cb18-29" tabindex="-1"></a><span class="co"># Evaluate on test data</span></span>
+<span id="cb18-30"><a href="#cb18-30" tabindex="-1"></a>test_labels, test_predictions <span class="op">=</span> evaluate_model(model, test_loader, device)</span>
+<span id="cb18-31"><a href="#cb18-31" tabindex="-1"></a>plot_confusion_matrix(test_labels, test_predictions, <span class="st">"Confusion Matrix for Test Data"</span>)</span></code></pre>
+</div>
+<div class="codewrapper sourceCode" id="cb19">
+<h3 class="code-label">PYTHON<i aria-hidden="true" data-feather="chevron-left"></i><i aria-hidden="true" data-feather="chevron-right"></i>
+</h3>
+<pre class="sourceCode python" tabindex="0"><code class="sourceCode python"><span id="cb19-1"><a href="#cb19-1" tabindex="-1"></a><span class="im">from</span> scipy.stats <span class="im">import</span> gaussian_kde</span>
+<span id="cb19-2"><a href="#cb19-2" tabindex="-1"></a><span class="im">from</span> pytorch_ood.detector <span class="im">import</span> EnergyBased</span>
+<span id="cb19-3"><a href="#cb19-3" tabindex="-1"></a><span class="im">from</span> sklearn.metrics <span class="im">import</span> precision_recall_fscore_support, accuracy_score</span>
+<span id="cb19-4"><a href="#cb19-4" tabindex="-1"></a></span>
+<span id="cb19-5"><a href="#cb19-5" tabindex="-1"></a><span class="co"># Compute softmax scores</span></span>
+<span id="cb19-6"><a href="#cb19-6" tabindex="-1"></a><span class="kw">def</span> get_softmax_scores(model, dataloader):</span>
+<span id="cb19-7"><a href="#cb19-7" tabindex="-1"></a>    model.<span class="bu">eval</span>()</span>
+<span id="cb19-8"><a href="#cb19-8" tabindex="-1"></a>    softmax_scores <span class="op">=</span> []</span>
+<span id="cb19-9"><a href="#cb19-9" tabindex="-1"></a>    <span class="cf">with</span> torch.no_grad():</span>
+<span id="cb19-10"><a href="#cb19-10" tabindex="-1"></a>        <span class="cf">for</span> inputs, _ <span class="kw">in</span> dataloader:</span>
+<span id="cb19-11"><a href="#cb19-11" tabindex="-1"></a>            inputs <span class="op">=</span> inputs.to(device)</span>
+<span id="cb19-12"><a href="#cb19-12" tabindex="-1"></a>            outputs <span class="op">=</span> model(inputs)</span>
+<span id="cb19-13"><a href="#cb19-13" tabindex="-1"></a>            softmax <span class="op">=</span> torch.nn.functional.softmax(outputs, dim<span class="op">=</span><span class="dv">1</span>)</span>
+<span id="cb19-14"><a href="#cb19-14" tabindex="-1"></a>            softmax_scores.extend(softmax.cpu().numpy())</span>
+<span id="cb19-15"><a href="#cb19-15" tabindex="-1"></a>    <span class="cf">return</span> np.array(softmax_scores)</span>
+<span id="cb19-16"><a href="#cb19-16" tabindex="-1"></a></span>
+<span id="cb19-17"><a href="#cb19-17" tabindex="-1"></a>id_softmax_scores <span class="op">=</span> get_softmax_scores(model, test_loader)</span>
+<span id="cb19-18"><a href="#cb19-18" tabindex="-1"></a>ood_softmax_scores <span class="op">=</span> get_softmax_scores(model, ood_loader)</span>
+<span id="cb19-19"><a href="#cb19-19" tabindex="-1"></a></span>
+<span id="cb19-20"><a href="#cb19-20" tabindex="-1"></a><span class="co"># Initialize the energy-based OOD detector</span></span>
+<span id="cb19-21"><a href="#cb19-21" tabindex="-1"></a>energy_detector <span class="op">=</span> EnergyBased(model, t<span class="op">=</span><span class="fl">1.0</span>)</span>
+<span id="cb19-22"><a href="#cb19-22" tabindex="-1"></a></span>
+<span id="cb19-23"><a href="#cb19-23" tabindex="-1"></a><span class="co"># Compute energy scores</span></span>
+<span id="cb19-24"><a href="#cb19-24" tabindex="-1"></a><span class="kw">def</span> get_energy_scores(detector, dataloader):</span>
+<span id="cb19-25"><a href="#cb19-25" tabindex="-1"></a>    scores <span class="op">=</span> []</span>
+<span id="cb19-26"><a href="#cb19-26" tabindex="-1"></a>    detector.model.<span class="bu">eval</span>()</span>
+<span id="cb19-27"><a href="#cb19-27" tabindex="-1"></a>    <span class="cf">with</span> torch.no_grad():</span>
+<span id="cb19-28"><a href="#cb19-28" tabindex="-1"></a>        <span class="cf">for</span> inputs, _ <span class="kw">in</span> dataloader:</span>
+<span id="cb19-29"><a href="#cb19-29" tabindex="-1"></a>            inputs <span class="op">=</span> inputs.to(device)</span>
+<span id="cb19-30"><a href="#cb19-30" tabindex="-1"></a>            score <span class="op">=</span> detector.predict(inputs)</span>
+<span id="cb19-31"><a href="#cb19-31" tabindex="-1"></a>            scores.extend(score.cpu().numpy())</span>
+<span id="cb19-32"><a href="#cb19-32" tabindex="-1"></a>    <span class="cf">return</span> np.array(scores)</span>
+<span id="cb19-33"><a href="#cb19-33" tabindex="-1"></a></span>
+<span id="cb19-34"><a href="#cb19-34" tabindex="-1"></a>id_energy_scores <span class="op">=</span> get_energy_scores(energy_detector, test_loader)</span>
+<span id="cb19-35"><a href="#cb19-35" tabindex="-1"></a>ood_energy_scores <span class="op">=</span> get_energy_scores(energy_detector, ood_loader)</span>
+<span id="cb19-36"><a href="#cb19-36" tabindex="-1"></a></span>
+<span id="cb19-37"><a href="#cb19-37" tabindex="-1"></a><span class="im">import</span> matplotlib.pyplot <span class="im">as</span> plt</span>
+<span id="cb19-38"><a href="#cb19-38" tabindex="-1"></a></span>
+<span id="cb19-39"><a href="#cb19-39" tabindex="-1"></a></span>
+<span id="cb19-40"><a href="#cb19-40" tabindex="-1"></a><span class="co"># Plot PSDs</span></span>
+<span id="cb19-41"><a href="#cb19-41" tabindex="-1"></a></span>
+<span id="cb19-42"><a href="#cb19-42" tabindex="-1"></a><span class="co"># Function to plot PSD</span></span>
+<span id="cb19-43"><a href="#cb19-43" tabindex="-1"></a><span class="kw">def</span> plot_psd(id_scores, ood_scores, method_name):</span>
+<span id="cb19-44"><a href="#cb19-44" tabindex="-1"></a>    plt.figure(figsize<span class="op">=</span>(<span class="dv">12</span>, <span class="dv">6</span>))</span>
+<span id="cb19-45"><a href="#cb19-45" tabindex="-1"></a>    alpha <span class="op">=</span> <span class="fl">0.3</span></span>
+<span id="cb19-46"><a href="#cb19-46" tabindex="-1"></a></span>
+<span id="cb19-47"><a href="#cb19-47" tabindex="-1"></a>    <span class="co"># Plot PSD for ID scores</span></span>
+<span id="cb19-48"><a href="#cb19-48" tabindex="-1"></a>    id_density <span class="op">=</span> gaussian_kde(id_scores)</span>
+<span id="cb19-49"><a href="#cb19-49" tabindex="-1"></a>    x_id <span class="op">=</span> np.linspace(id_scores.<span class="bu">min</span>(), id_scores.<span class="bu">max</span>(), <span class="dv">1000</span>)</span>
+<span id="cb19-50"><a href="#cb19-50" tabindex="-1"></a>    plt.plot(x_id, id_density(x_id), label<span class="op">=</span><span class="ss">f'ID (</span><span class="sc">{</span>method_name<span class="sc">}</span><span class="ss">)'</span>, color<span class="op">=</span><span class="st">'blue'</span>, alpha<span class="op">=</span>alpha)</span>
+<span id="cb19-51"><a href="#cb19-51" tabindex="-1"></a></span>
+<span id="cb19-52"><a href="#cb19-52" tabindex="-1"></a>    <span class="co"># Plot PSD for OOD scores</span></span>
+<span id="cb19-53"><a href="#cb19-53" tabindex="-1"></a>    ood_density <span class="op">=</span> gaussian_kde(ood_scores)</span>
+<span id="cb19-54"><a href="#cb19-54" tabindex="-1"></a>    x_ood <span class="op">=</span> np.linspace(ood_scores.<span class="bu">min</span>(), ood_scores.<span class="bu">max</span>(), <span class="dv">1000</span>)</span>
+<span id="cb19-55"><a href="#cb19-55" tabindex="-1"></a>    plt.plot(x_ood, ood_density(x_ood), label<span class="op">=</span><span class="ss">f'OOD (</span><span class="sc">{</span>method_name<span class="sc">}</span><span class="ss">)'</span>, color<span class="op">=</span><span class="st">'red'</span>, alpha<span class="op">=</span>alpha)</span>
+<span id="cb19-56"><a href="#cb19-56" tabindex="-1"></a></span>
+<span id="cb19-57"><a href="#cb19-57" tabindex="-1"></a>    plt.xlabel(<span class="st">'Score'</span>)</span>
+<span id="cb19-58"><a href="#cb19-58" tabindex="-1"></a>    plt.ylabel(<span class="st">'Density'</span>)</span>
+<span id="cb19-59"><a href="#cb19-59" tabindex="-1"></a>    plt.title(<span class="ss">f'Probability Density Distributions for </span><span class="sc">{</span>method_name<span class="sc">}</span><span class="ss"> Scores'</span>)</span>
+<span id="cb19-60"><a href="#cb19-60" tabindex="-1"></a>    plt.legend()</span>
+<span id="cb19-61"><a href="#cb19-61" tabindex="-1"></a>    plt.show()</span>
+<span id="cb19-62"><a href="#cb19-62" tabindex="-1"></a></span>
+<span id="cb19-63"><a href="#cb19-63" tabindex="-1"></a><span class="co"># Plot PSD for softmax scores</span></span>
+<span id="cb19-64"><a href="#cb19-64" tabindex="-1"></a>plot_psd(id_softmax_scores[:, <span class="dv">1</span>], ood_softmax_scores[:, <span class="dv">1</span>], <span class="st">'Softmax'</span>)</span>
+<span id="cb19-65"><a href="#cb19-65" tabindex="-1"></a></span>
+<span id="cb19-66"><a href="#cb19-66" tabindex="-1"></a><span class="co"># Plot PSD for energy scores</span></span>
+<span id="cb19-67"><a href="#cb19-67" tabindex="-1"></a>plot_psd(id_energy_scores, ood_energy_scores, <span class="st">'Energy'</span>)</span>
+<span id="cb19-68"><a href="#cb19-68" tabindex="-1"></a></span></code></pre>
+</div>
+<div class="codewrapper sourceCode" id="cb20">
+<h3 class="code-label">PYTHON<i aria-hidden="true" data-feather="chevron-left"></i><i aria-hidden="true" data-feather="chevron-right"></i>
+</h3>
+<pre class="sourceCode python" tabindex="0"><code class="sourceCode python"><span id="cb20-1"><a href="#cb20-1" tabindex="-1"></a><span class="im">import</span> numpy <span class="im">as</span> np</span>
+<span id="cb20-2"><a href="#cb20-2" tabindex="-1"></a><span class="im">import</span> matplotlib.pyplot <span class="im">as</span> plt</span>
+<span id="cb20-3"><a href="#cb20-3" tabindex="-1"></a><span class="im">from</span> sklearn.metrics <span class="im">import</span> precision_recall_fscore_support, accuracy_score, confusion_matrix, ConfusionMatrixDisplay</span>
+<span id="cb20-4"><a href="#cb20-4" tabindex="-1"></a></span>
+<span id="cb20-5"><a href="#cb20-5" tabindex="-1"></a><span class="co"># Define thresholds to evaluate</span></span>
+<span id="cb20-6"><a href="#cb20-6" tabindex="-1"></a>thresholds <span class="op">=</span> np.linspace(id_energy_scores.<span class="bu">min</span>(), id_energy_scores.<span class="bu">max</span>(), <span class="dv">50</span>)</span>
+<span id="cb20-7"><a href="#cb20-7" tabindex="-1"></a></span>
+<span id="cb20-8"><a href="#cb20-8" tabindex="-1"></a><span class="co"># Store evaluation metrics for each threshold</span></span>
+<span id="cb20-9"><a href="#cb20-9" tabindex="-1"></a>accuracies <span class="op">=</span> []</span>
+<span id="cb20-10"><a href="#cb20-10" tabindex="-1"></a>precisions <span class="op">=</span> []</span>
+<span id="cb20-11"><a href="#cb20-11" tabindex="-1"></a>recalls <span class="op">=</span> []</span>
+<span id="cb20-12"><a href="#cb20-12" tabindex="-1"></a>f1_scores <span class="op">=</span> []</span>
+<span id="cb20-13"><a href="#cb20-13" tabindex="-1"></a></span>
+<span id="cb20-14"><a href="#cb20-14" tabindex="-1"></a><span class="co"># True labels for OOD data (since they are not part of the original labels)</span></span>
+<span id="cb20-15"><a href="#cb20-15" tabindex="-1"></a>ood_true_labels <span class="op">=</span> np.full(<span class="bu">len</span>(ood_energy_scores), <span class="op">-</span><span class="dv">1</span>)</span>
+<span id="cb20-16"><a href="#cb20-16" tabindex="-1"></a></span>
+<span id="cb20-17"><a href="#cb20-17" tabindex="-1"></a><span class="co"># We need the test_labels to be aligned with the ID data</span></span>
+<span id="cb20-18"><a href="#cb20-18" tabindex="-1"></a>id_true_labels <span class="op">=</span> test_labels[:<span class="bu">len</span>(id_energy_scores)]</span>
+<span id="cb20-19"><a href="#cb20-19" tabindex="-1"></a></span>
+<span id="cb20-20"><a href="#cb20-20" tabindex="-1"></a><span class="cf">for</span> threshold <span class="kw">in</span> thresholds:</span>
+<span id="cb20-21"><a href="#cb20-21" tabindex="-1"></a>    <span class="co"># Classify OOD examples based on energy scores</span></span>
+<span id="cb20-22"><a href="#cb20-22" tabindex="-1"></a>    ood_classifications <span class="op">=</span> np.where(ood_energy_scores <span class="op">&gt;=</span> threshold, <span class="op">-</span><span class="dv">1</span>,  <span class="co"># classified as OOD</span></span>
+<span id="cb20-23"><a href="#cb20-23" tabindex="-1"></a>                                   np.where(ood_energy_scores <span class="op">&lt;</span> threshold, <span class="dv">0</span>, <span class="op">-</span><span class="dv">1</span>))  <span class="co"># classified as ID</span></span>
+<span id="cb20-24"><a href="#cb20-24" tabindex="-1"></a></span>
+<span id="cb20-25"><a href="#cb20-25" tabindex="-1"></a>    <span class="co"># Classify ID examples based on energy scores</span></span>
+<span id="cb20-26"><a href="#cb20-26" tabindex="-1"></a>    id_classifications <span class="op">=</span> np.where(id_energy_scores <span class="op">&gt;=</span> threshold, <span class="op">-</span><span class="dv">1</span>,  <span class="co"># classified as OOD</span></span>
+<span id="cb20-27"><a href="#cb20-27" tabindex="-1"></a>                                  np.where(id_energy_scores <span class="op">&lt;</span> threshold, id_true_labels, <span class="op">-</span><span class="dv">1</span>))  <span class="co"># classified as ID</span></span>
+<span id="cb20-28"><a href="#cb20-28" tabindex="-1"></a></span>
+<span id="cb20-29"><a href="#cb20-29" tabindex="-1"></a>    <span class="co"># Combine OOD and ID classifications and true labels</span></span>
+<span id="cb20-30"><a href="#cb20-30" tabindex="-1"></a>    all_predictions <span class="op">=</span> np.concatenate([ood_classifications, id_classifications])</span>
+<span id="cb20-31"><a href="#cb20-31" tabindex="-1"></a>    all_true_labels <span class="op">=</span> np.concatenate([ood_true_labels, id_true_labels])</span>
+<span id="cb20-32"><a href="#cb20-32" tabindex="-1"></a></span>
+<span id="cb20-33"><a href="#cb20-33" tabindex="-1"></a>    <span class="co"># Evaluate metrics</span></span>
+<span id="cb20-34"><a href="#cb20-34" tabindex="-1"></a>    precision, recall, f1, _ <span class="op">=</span> precision_recall_fscore_support(all_true_labels, all_predictions, labels<span class="op">=</span>[<span class="dv">0</span>, <span class="dv">1</span>], average<span class="op">=</span><span class="st">'macro'</span>, zero_division<span class="op">=</span><span class="dv">0</span>)</span>
+<span id="cb20-35"><a href="#cb20-35" tabindex="-1"></a>    accuracy <span class="op">=</span> accuracy_score(all_true_labels, all_predictions)</span>
+<span id="cb20-36"><a href="#cb20-36" tabindex="-1"></a></span>
+<span id="cb20-37"><a href="#cb20-37" tabindex="-1"></a>    accuracies.append(accuracy)</span>
+<span id="cb20-38"><a href="#cb20-38" tabindex="-1"></a>    precisions.append(precision)</span>
+<span id="cb20-39"><a href="#cb20-39" tabindex="-1"></a>    recalls.append(recall)</span>
+<span id="cb20-40"><a href="#cb20-40" tabindex="-1"></a>    f1_scores.append(f1)</span>
+<span id="cb20-41"><a href="#cb20-41" tabindex="-1"></a></span>
+<span id="cb20-42"><a href="#cb20-42" tabindex="-1"></a><span class="co"># Find the best thresholds for each metric</span></span>
+<span id="cb20-43"><a href="#cb20-43" tabindex="-1"></a>best_f1_index <span class="op">=</span> np.argmax(f1_scores)</span>
+<span id="cb20-44"><a href="#cb20-44" tabindex="-1"></a>best_f1_threshold <span class="op">=</span> thresholds[best_f1_index]</span>
+<span id="cb20-45"><a href="#cb20-45" tabindex="-1"></a></span>
+<span id="cb20-46"><a href="#cb20-46" tabindex="-1"></a>best_precision_index <span class="op">=</span> np.argmax(precisions)</span>
+<span id="cb20-47"><a href="#cb20-47" tabindex="-1"></a>best_precision_threshold <span class="op">=</span> thresholds[best_precision_index]</span>
+<span id="cb20-48"><a href="#cb20-48" tabindex="-1"></a></span>
+<span id="cb20-49"><a href="#cb20-49" tabindex="-1"></a>best_recall_index <span class="op">=</span> np.argmax(recalls)</span>
+<span id="cb20-50"><a href="#cb20-50" tabindex="-1"></a>best_recall_threshold <span class="op">=</span> thresholds[best_recall_index]</span>
+<span id="cb20-51"><a href="#cb20-51" tabindex="-1"></a></span>
+<span id="cb20-52"><a href="#cb20-52" tabindex="-1"></a><span class="bu">print</span>(<span class="ss">f"Best F1 threshold: </span><span class="sc">{</span>best_f1_threshold<span class="sc">}</span><span class="ss">, F1 Score: </span><span class="sc">{</span>f1_scores[best_f1_index]<span class="sc">}</span><span class="ss">"</span>)</span>
+<span id="cb20-53"><a href="#cb20-53" tabindex="-1"></a><span class="bu">print</span>(<span class="ss">f"Best Precision threshold: </span><span class="sc">{</span>best_precision_threshold<span class="sc">}</span><span class="ss">, Precision: </span><span class="sc">{</span>precisions[best_precision_index]<span class="sc">}</span><span class="ss">"</span>)</span>
+<span id="cb20-54"><a href="#cb20-54" tabindex="-1"></a><span class="bu">print</span>(<span class="ss">f"Best Recall threshold: </span><span class="sc">{</span>best_recall_threshold<span class="sc">}</span><span class="ss">, Recall: </span><span class="sc">{</span>recalls[best_recall_index]<span class="sc">}</span><span class="ss">"</span>)</span>
+<span id="cb20-55"><a href="#cb20-55" tabindex="-1"></a></span>
+<span id="cb20-56"><a href="#cb20-56" tabindex="-1"></a><span class="co"># Plot metrics as functions of the threshold</span></span>
+<span id="cb20-57"><a href="#cb20-57" tabindex="-1"></a>plt.figure(figsize<span class="op">=</span>(<span class="dv">12</span>, <span class="dv">8</span>))</span>
+<span id="cb20-58"><a href="#cb20-58" tabindex="-1"></a>plt.plot(thresholds, precisions, label<span class="op">=</span><span class="st">'Precision'</span>, color<span class="op">=</span><span class="st">'g'</span>)</span>
+<span id="cb20-59"><a href="#cb20-59" tabindex="-1"></a>plt.plot(thresholds, recalls, label<span class="op">=</span><span class="st">'Recall'</span>, color<span class="op">=</span><span class="st">'b'</span>)</span>
+<span id="cb20-60"><a href="#cb20-60" tabindex="-1"></a>plt.plot(thresholds, f1_scores, label<span class="op">=</span><span class="st">'F1 Score'</span>, color<span class="op">=</span><span class="st">'r'</span>)</span>
+<span id="cb20-61"><a href="#cb20-61" tabindex="-1"></a></span>
+<span id="cb20-62"><a href="#cb20-62" tabindex="-1"></a><span class="co"># Add best threshold indicators</span></span>
+<span id="cb20-63"><a href="#cb20-63" tabindex="-1"></a>plt.axvline(x<span class="op">=</span>best_f1_threshold, color<span class="op">=</span><span class="st">'r'</span>, linestyle<span class="op">=</span><span class="st">'--'</span>, label<span class="op">=</span><span class="ss">f'Best F1 Threshold: </span><span class="sc">{</span>best_f1_threshold<span class="sc">:.2f}</span><span class="ss">'</span>)</span>
+<span id="cb20-64"><a href="#cb20-64" tabindex="-1"></a>plt.axvline(x<span class="op">=</span>best_precision_threshold, color<span class="op">=</span><span class="st">'g'</span>, linestyle<span class="op">=</span><span class="st">'--'</span>, label<span class="op">=</span><span class="ss">f'Best Precision Threshold: </span><span class="sc">{</span>best_precision_threshold<span class="sc">:.2f}</span><span class="ss">'</span>)</span>
+<span id="cb20-65"><a href="#cb20-65" tabindex="-1"></a>plt.axvline(x<span class="op">=</span>best_recall_threshold, color<span class="op">=</span><span class="st">'b'</span>, linestyle<span class="op">=</span><span class="st">'--'</span>, label<span class="op">=</span><span class="ss">f'Best Recall Threshold: </span><span class="sc">{</span>best_recall_threshold<span class="sc">:.2f}</span><span class="ss">'</span>)</span>
+<span id="cb20-66"><a href="#cb20-66" tabindex="-1"></a></span>
+<span id="cb20-67"><a href="#cb20-67" tabindex="-1"></a>plt.xlabel(<span class="st">'Threshold'</span>)</span>
+<span id="cb20-68"><a href="#cb20-68" tabindex="-1"></a>plt.ylabel(<span class="st">'Metric Value'</span>)</span>
+<span id="cb20-69"><a href="#cb20-69" tabindex="-1"></a>plt.title(<span class="st">'Evaluation Metrics as Functions of Threshold (Energy-Based OOD Detection)'</span>)</span>
+<span id="cb20-70"><a href="#cb20-70" tabindex="-1"></a>plt.legend()</span>
+<span id="cb20-71"><a href="#cb20-71" tabindex="-1"></a>plt.show()</span></code></pre>
+</div>
+<div class="codewrapper sourceCode" id="cb21">
+<h3 class="code-label">PYTHON<i aria-hidden="true" data-feather="chevron-left"></i><i aria-hidden="true" data-feather="chevron-right"></i>
+</h3>
+<pre class="sourceCode python" tabindex="0"><code class="sourceCode python"><span id="cb21-1"><a href="#cb21-1" tabindex="-1"></a><span class="im">import</span> numpy <span class="im">as</span> np</span>
+<span id="cb21-2"><a href="#cb21-2" tabindex="-1"></a><span class="im">import</span> matplotlib.pyplot <span class="im">as</span> plt</span>
+<span id="cb21-3"><a href="#cb21-3" tabindex="-1"></a><span class="im">from</span> sklearn.metrics <span class="im">import</span> confusion_matrix, ConfusionMatrixDisplay</span>
+<span id="cb21-4"><a href="#cb21-4" tabindex="-1"></a></span>
+<span id="cb21-5"><a href="#cb21-5" tabindex="-1"></a><span class="co"># Threshold value for the energy score</span></span>
+<span id="cb21-6"><a href="#cb21-6" tabindex="-1"></a>upper_threshold <span class="op">=</span> best_f1_threshold  <span class="co"># Using the best F1 threshold from the previous calculation</span></span>
+<span id="cb21-7"><a href="#cb21-7" tabindex="-1"></a></span>
+<span id="cb21-8"><a href="#cb21-8" tabindex="-1"></a><span class="co"># Classifying OOD examples based on energy scores</span></span>
+<span id="cb21-9"><a href="#cb21-9" tabindex="-1"></a>ood_classifications <span class="op">=</span> np.where(ood_energy_scores <span class="op">&gt;=</span> upper_threshold, <span class="op">-</span><span class="dv">1</span>,  <span class="co"># classified as OOD</span></span>
+<span id="cb21-10"><a href="#cb21-10" tabindex="-1"></a>                               np.where(ood_energy_scores <span class="op">&lt;</span> upper_threshold, <span class="dv">0</span>, <span class="op">-</span><span class="dv">1</span>))  <span class="co"># classified as ID</span></span>
+<span id="cb21-11"><a href="#cb21-11" tabindex="-1"></a></span>
+<span id="cb21-12"><a href="#cb21-12" tabindex="-1"></a><span class="co"># Classifying ID examples based on energy scores</span></span>
+<span id="cb21-13"><a href="#cb21-13" tabindex="-1"></a>id_classifications <span class="op">=</span> np.where(id_energy_scores <span class="op">&gt;=</span> upper_threshold, <span class="op">-</span><span class="dv">1</span>,  <span class="co"># classified as OOD</span></span>
+<span id="cb21-14"><a href="#cb21-14" tabindex="-1"></a>                              np.where(id_energy_scores <span class="op">&lt;</span> upper_threshold, id_true_labels, <span class="op">-</span><span class="dv">1</span>))  <span class="co"># classified as ID</span></span>
+<span id="cb21-15"><a href="#cb21-15" tabindex="-1"></a></span>
+<span id="cb21-16"><a href="#cb21-16" tabindex="-1"></a><span class="co"># Combine OOD and ID classifications and true labels</span></span>
+<span id="cb21-17"><a href="#cb21-17" tabindex="-1"></a>all_predictions <span class="op">=</span> np.concatenate([ood_classifications, id_classifications])</span>
+<span id="cb21-18"><a href="#cb21-18" tabindex="-1"></a>all_true_labels <span class="op">=</span> np.concatenate([ood_true_labels, id_true_labels])</span>
+<span id="cb21-19"><a href="#cb21-19" tabindex="-1"></a></span>
+<span id="cb21-20"><a href="#cb21-20" tabindex="-1"></a><span class="co"># Confusion matrix</span></span>
+<span id="cb21-21"><a href="#cb21-21" tabindex="-1"></a>cm <span class="op">=</span> confusion_matrix(all_true_labels, all_predictions, labels<span class="op">=</span>[<span class="dv">0</span>, <span class="dv">1</span>, <span class="op">-</span><span class="dv">1</span>])</span>
+<span id="cb21-22"><a href="#cb21-22" tabindex="-1"></a></span>
+<span id="cb21-23"><a href="#cb21-23" tabindex="-1"></a><span class="co"># Plotting the confusion matrix</span></span>
+<span id="cb21-24"><a href="#cb21-24" tabindex="-1"></a>disp <span class="op">=</span> ConfusionMatrixDisplay(confusion_matrix<span class="op">=</span>cm, display_labels<span class="op">=</span>[<span class="st">"Shirt"</span>, <span class="st">"Pants"</span>, <span class="st">"OOD"</span>])</span>
+<span id="cb21-25"><a href="#cb21-25" tabindex="-1"></a>disp.plot(cmap<span class="op">=</span>plt.cm.Blues)</span>
+<span id="cb21-26"><a href="#cb21-26" tabindex="-1"></a>plt.title(<span class="st">'Confusion Matrix for OOD and ID Classification (Energy-Based)'</span>)</span>
+<span id="cb21-27"><a href="#cb21-27" tabindex="-1"></a>plt.show()</span></code></pre>
+</div>
+</div>
+</div>
+</div>
+<div class="section level1">
+<h1 id="conclusion">Conclusion<a class="anchor" aria-label="anchor" href="#conclusion"></a></h1>
+<div class="codewrapper sourceCode" id="cb22">
 <h3 class="code-label">PYTHON<i aria-hidden="true" data-feather="chevron-left"></i><i aria-hidden="true" data-feather="chevron-right"></i>
 </h3>
 <pre class="sourceCode python" tabindex="0"><code class="sourceCode python"></code></pre>
 </div>
-<!--
+<div class="section level2">
+<h2 id="references-and-supplemental-resources">References and supplemental resources<a class="anchor" aria-label="anchor" href="#references-and-supplemental-resources"></a></h2>
+<ul><li><a href="https://www.youtube.com/watch?v=hgLC9_9ZCJI" class="external-link uri">https://www.youtube.com/watch?v=hgLC9_9ZCJI</a></li>
+<li>Generalized Out-of-Distribution Detection: A Survey: <a href="https://arxiv.org/abs/2110.11334" class="external-link uri">https://arxiv.org/abs/2110.11334</a> # Glossary</li>
+<li>ID/OOD: In-distribution, out-of-distribution. Generally, the OOD
+instances can be defined as instances (x, y) sampled from an underlying
+distribution other than the training distribution P(Xtrain, Ytrain),
+where Xtrain and Ytrain are the training corpus and training label set,
+respectively.</li>
+<li>OOD instances with semantic shift: OOD instances with semantic shift
+refer to instances that do not belong to y_train. More specifically,
+instances with semantic shift may come from unknown categories or
+irrelevant tasks.</li>
+<li>OOD instances with covariate shift: OOD instances with non-semantic
+shift refer to the instances that belong to y_train but are sampled from
+a distribution other than x_train, e.g., a different
+domain/corpus/location.</li>
+<li>Closed-world assumption: an assumption that the training and test
+data are sampled from the same distribution. However, training data can
+rarely capture the entire distribution. In real-world scenarios,
+out-of-distribution (OOD) instances, which come from categories that are
+not known to the model, can often be present in inference phases.</li>
+<li>Inference-time OOD: After training, use some kind of scoring
+function to determine if test inputs are OOD or not.</li>
+<li>Output-based OOD: Output-based OOD detection methods leverage the
+model’s output distribution to identify OOD instances. These methods
+typically involve analyzing the softmax scores, confidence scores, or
+other output statistics to detect anomalies.</li>
+</ul><!--
 Place links that you need to refer to multiple times across pages here. Delete
 any links that you are not going to use.
- -->
+ --></div>
 </div>
 
 
diff --git a/instructor/aio.html b/instructor/aio.html
index 46b4438d..5742dc58 100644
--- a/instructor/aio.html
+++ b/instructor/aio.html
@@ -595,7 +595,7 @@ <h3 class="callout-title">Challenge<a class="anchor" aria-label="anchor" href="#
 <button class="accordion-button solution-button collapsed" type="button" data-bs-toggle="collapse" data-bs-target="#collapseSolution1" aria-expanded="false" aria-controls="collapseSolution1">
   <h4 class="accordion-header" id="headingSolution1"> Show me the solution </h4>
 </button>
-<div id="collapseSolution1" class="accordion-collapse collapse" aria-labelledby="headingSolution1" data-bs-parent="#accordionSolution1">
+<div id="collapseSolution1" class="accordion-collapse collapse" data-bs-parent="#accordionSolution1" aria-labelledby="headingSolution1">
 <div class="accordion-body">
 <p>A summary of the principles is listed below:</p>
 <ul>
@@ -3108,7 +3108,7 @@ <h3 class="callout-title">Classifying explanation techniques<a class="anchor" ar
 <button class="accordion-button solution-button collapsed" type="button" data-bs-toggle="collapse" data-bs-target="#collapseSolution1" aria-expanded="false" aria-controls="collapseSolution1">
   <h4 class="accordion-header" id="headingSolution1"> Show me the solution </h4>
 </button>
-<div id="collapseSolution1" class="accordion-collapse collapse" data-bs-parent="#accordionSolution1" aria-labelledby="headingSolution1">
+<div id="collapseSolution1" class="accordion-collapse collapse" aria-labelledby="headingSolution1" data-bs-parent="#accordionSolution1">
 <div class="accordion-body">
 <table class="table">
 <colgroup>
@@ -4292,8 +4292,8 @@ <h3 class="card-title">Questions</h3>
 <ul>
 <li>What are out-of-distribution (OOD) data and why is detecting them
 important in machine learning models?</li>
-<li>How do output-based methods like softmax, energy-based, and
-distance-based methods work for OOD detection?</li>
+<li>How do output-based methods like softmax and energy-based methods
+work for OOD detection?</li>
 <li>What are the limitations of output-based OOD detection methods?</li>
 </ul>
 </div>
@@ -4902,7 +4902,7 @@ <h3 class="code-label">PYTHON<i aria-hidden="true" data-feather="chevron-left"><
 <span id="cb14-29"><a href="#cb14-29" tabindex="-1"></a>  all_true_labels <span class="op">=</span> np.concatenate([<span class="op">-</span><span class="dv">1</span> <span class="op">*</span> np.ones(ood_classifications.shape), train_labels])</span>
 <span id="cb14-30"><a href="#cb14-30" tabindex="-1"></a></span>
 <span id="cb14-31"><a href="#cb14-31" tabindex="-1"></a>  <span class="co"># Evaluate metrics</span></span>
-<span id="cb14-32"><a href="#cb14-32" tabindex="-1"></a>  precision, recall, f1, _ <span class="op">=</span> precision_recall_fscore_support(all_true_labels, all_predictions, labels<span class="op">=</span>[<span class="dv">0</span>, <span class="dv">1</span>], average<span class="op">=</span><span class="st">'macro'</span>) <span class="co"># discuss macro vs micro .</span></span>
+<span id="cb14-32"><a href="#cb14-32" tabindex="-1"></a>  precision, recall, f1, _ <span class="op">=</span> precision_recall_fscore_support(all_true_labels, all_predictions, labels<span class="op">=</span>[<span class="dv">0</span>, <span class="dv">1</span>, <span class="op">-</span><span class="dv">1</span>], average<span class="op">=</span><span class="st">'macro'</span>) <span class="co"># discuss macro vs micro .</span></span>
 <span id="cb14-33"><a href="#cb14-33" tabindex="-1"></a>  accuracy <span class="op">=</span> accuracy_score(all_true_labels, all_predictions)</span>
 <span id="cb14-34"><a href="#cb14-34" tabindex="-1"></a></span>
 <span id="cb14-35"><a href="#cb14-35" tabindex="-1"></a>  accuracies.append(accuracy)</span>
@@ -4941,14 +4941,52 @@ <h3 class="code-label">PYTHON<i aria-hidden="true" data-feather="chevron-left"><
 <span id="cb14-68"><a href="#cb14-68" tabindex="-1"></a>plt.legend()</span>
 <span id="cb14-69"><a href="#cb14-69" tabindex="-1"></a>plt.show()</span></code></pre>
 </div>
+<div class="codewrapper sourceCode" id="cb15">
+<h3 class="code-label">PYTHON<i aria-hidden="true" data-feather="chevron-left"></i><i aria-hidden="true" data-feather="chevron-right"></i>
+</h3>
+<pre class="sourceCode python" tabindex="0"><code class="sourceCode python"><span id="cb15-1"><a href="#cb15-1" tabindex="-1"></a><span class="im">import</span> numpy <span class="im">as</span> np</span>
+<span id="cb15-2"><a href="#cb15-2" tabindex="-1"></a><span class="im">import</span> matplotlib.pyplot <span class="im">as</span> plt</span>
+<span id="cb15-3"><a href="#cb15-3" tabindex="-1"></a><span class="im">from</span> sklearn.metrics <span class="im">import</span> confusion_matrix, ConfusionMatrixDisplay</span>
+<span id="cb15-4"><a href="#cb15-4" tabindex="-1"></a></span>
+<span id="cb15-5"><a href="#cb15-5" tabindex="-1"></a><span class="co"># Assuming ood_probs, id_probs, and train_labels are defined</span></span>
+<span id="cb15-6"><a href="#cb15-6" tabindex="-1"></a><span class="co"># Threshold values</span></span>
+<span id="cb15-7"><a href="#cb15-7" tabindex="-1"></a>upper_threshold <span class="op">=</span> best_f1_threshold</span>
+<span id="cb15-8"><a href="#cb15-8" tabindex="-1"></a></span>
+<span id="cb15-9"><a href="#cb15-9" tabindex="-1"></a><span class="co"># Classifying OOD examples (sandals)</span></span>
+<span id="cb15-10"><a href="#cb15-10" tabindex="-1"></a>ood_classifications <span class="op">=</span> np.where(ood_probs[:, <span class="dv">1</span>] <span class="op">&gt;=</span> upper_threshold, <span class="dv">1</span>,  <span class="co"># classified as pants</span></span>
+<span id="cb15-11"><a href="#cb15-11" tabindex="-1"></a>                               np.where(ood_probs[:, <span class="dv">0</span>] <span class="op">&gt;=</span> upper_threshold, <span class="dv">0</span>,  <span class="co"># classified as shirts</span></span>
+<span id="cb15-12"><a href="#cb15-12" tabindex="-1"></a>                                        <span class="op">-</span><span class="dv">1</span>))  <span class="co"># classified as OOD</span></span>
+<span id="cb15-13"><a href="#cb15-13" tabindex="-1"></a>ood_classifications</span>
+<span id="cb15-14"><a href="#cb15-14" tabindex="-1"></a></span>
+<span id="cb15-15"><a href="#cb15-15" tabindex="-1"></a>id_probs</span>
+<span id="cb15-16"><a href="#cb15-16" tabindex="-1"></a><span class="co"># Classifying ID examples (T-shirts and pants)</span></span>
+<span id="cb15-17"><a href="#cb15-17" tabindex="-1"></a>id_classifications <span class="op">=</span> np.where(id_probs[:, <span class="dv">1</span>] <span class="op">&gt;=</span> upper_threshold, <span class="dv">1</span>,  <span class="co"># classified as pants</span></span>
+<span id="cb15-18"><a href="#cb15-18" tabindex="-1"></a>                              np.where(id_probs[:, <span class="dv">0</span>] <span class="op">&gt;=</span> upper_threshold, <span class="dv">0</span>,  <span class="co"># classified as shirts</span></span>
+<span id="cb15-19"><a href="#cb15-19" tabindex="-1"></a>                                       <span class="op">-</span><span class="dv">1</span>))  <span class="co"># classified as OOD</span></span>
+<span id="cb15-20"><a href="#cb15-20" tabindex="-1"></a></span>
+<span id="cb15-21"><a href="#cb15-21" tabindex="-1"></a>id_classifications</span>
+<span id="cb15-22"><a href="#cb15-22" tabindex="-1"></a></span>
+<span id="cb15-23"><a href="#cb15-23" tabindex="-1"></a><span class="co"># Combine OOD and ID classifications and true labels</span></span>
+<span id="cb15-24"><a href="#cb15-24" tabindex="-1"></a>all_predictions <span class="op">=</span> np.concatenate([ood_classifications, id_classifications])</span>
+<span id="cb15-25"><a href="#cb15-25" tabindex="-1"></a>all_true_labels <span class="op">=</span> np.concatenate([<span class="op">-</span><span class="dv">1</span> <span class="op">*</span> np.ones(ood_classifications.shape), train_labels])</span>
+<span id="cb15-26"><a href="#cb15-26" tabindex="-1"></a></span>
+<span id="cb15-27"><a href="#cb15-27" tabindex="-1"></a><span class="co"># Confusion matrix</span></span>
+<span id="cb15-28"><a href="#cb15-28" tabindex="-1"></a>cm <span class="op">=</span> confusion_matrix(all_true_labels, all_predictions, labels<span class="op">=</span>[<span class="dv">0</span>, <span class="dv">1</span>, <span class="op">-</span><span class="dv">1</span>])</span>
+<span id="cb15-29"><a href="#cb15-29" tabindex="-1"></a></span>
+<span id="cb15-30"><a href="#cb15-30" tabindex="-1"></a><span class="co"># Plotting the confusion matrix</span></span>
+<span id="cb15-31"><a href="#cb15-31" tabindex="-1"></a>disp <span class="op">=</span> ConfusionMatrixDisplay(confusion_matrix<span class="op">=</span>cm, display_labels<span class="op">=</span>[<span class="st">"Shirt"</span>, <span class="st">"Pants"</span>, <span class="st">"OOD"</span>])</span>
+<span id="cb15-32"><a href="#cb15-32" tabindex="-1"></a>disp.plot(cmap<span class="op">=</span>plt.cm.Blues)</span>
+<span id="cb15-33"><a href="#cb15-33" tabindex="-1"></a>plt.title(<span class="st">'Confusion Matrix for OOD and ID Classification'</span>)</span>
+<span id="cb15-34"><a href="#cb15-34" tabindex="-1"></a>plt.show()</span></code></pre>
+</div>
 </div>
 </div>
 </div>
 <div class="section level1">
 <h1 id="example-2-energy-based-ood-detection">Example 2: Energy-Based OOD Detection<a class="anchor" aria-label="anchor" href="#example-2-energy-based-ood-detection"></a>
 </h1>
-<p>Liu et al., Energy-based Out-of-distribution Detection, NeurIPS
-2020</p>
+<p>Liu et al., Energy-based Out-of-distribution Detection, NeurIPS 2020;
+<a href="https://arxiv.org/pdf/2010.03759" class="external-link uri">https://arxiv.org/pdf/2010.03759</a></p>
 <ul>
 <li><p>E(x, y) = energy value</p></li>
 <li><p>if x and y are “compatitble”, lower energy</p></li>
@@ -4961,17 +4999,377 @@ <h1 id="example-2-energy-based-ood-detection">Example 2: Energy-Based OOD Detect
 </li>
 <li><p>With energy scores, ID and OOD distributions become much more
 separable</p></li>
-<li><p>Another “output-based” method like softmax # Conclusion</p></li>
+<li><p>Another “output-based” method like softmax</p></li>
 </ul>
-<div class="codewrapper sourceCode" id="cb15">
+<div class="section level2">
+<h2 id="pytorch-out-of-distribution-detection">PyTorch Out-of-Distribution Detection<a class="anchor" aria-label="anchor" href="#pytorch-out-of-distribution-detection"></a>
+</h2>
+<p>There’s a Pytorch package for OOD detection! <a href="https://pytorch-ood.readthedocs.io/en/latest/info.html" class="external-link uri">https://pytorch-ood.readthedocs.io/en/latest/info.html</a></p>
+<div class="codewrapper sourceCode" id="cb16">
+<h3 class="code-label">PYTHON<i aria-hidden="true" data-feather="chevron-left"></i><i aria-hidden="true" data-feather="chevron-right"></i>
+</h3>
+<pre class="sourceCode python" tabindex="0"><code class="sourceCode python"><span id="cb16-1"><a href="#cb16-1" tabindex="-1"></a><span class="op">!</span>pip install pytorch<span class="op">-</span>ood</span></code></pre>
+</div>
+<div class="section level3">
+<h3 id="energy-based-is-designed-to-work-with-neural-nets-unpack-this-">Energy-based is designed to work with neural nets… unpack this.<a class="anchor" aria-label="anchor" href="#energy-based-is-designed-to-work-with-neural-nets-unpack-this-"></a>
+</h3>
+<p>Let’s train a simple CNN model on the FashionMNIST dataset.</p>
+<div class="codewrapper sourceCode" id="cb17">
+<h3 class="code-label">PYTHON<i aria-hidden="true" data-feather="chevron-left"></i><i aria-hidden="true" data-feather="chevron-right"></i>
+</h3>
+<pre class="sourceCode python" tabindex="0"><code class="sourceCode python"><span id="cb17-1"><a href="#cb17-1" tabindex="-1"></a><span class="im">import</span> torch</span>
+<span id="cb17-2"><a href="#cb17-2" tabindex="-1"></a><span class="im">import</span> torch.nn <span class="im">as</span> nn</span>
+<span id="cb17-3"><a href="#cb17-3" tabindex="-1"></a><span class="im">import</span> torch.optim <span class="im">as</span> optim</span>
+<span id="cb17-4"><a href="#cb17-4" tabindex="-1"></a><span class="im">import</span> torchvision.transforms <span class="im">as</span> transforms</span>
+<span id="cb17-5"><a href="#cb17-5" tabindex="-1"></a><span class="im">from</span> keras.datasets <span class="im">import</span> fashion_mnist</span>
+<span id="cb17-6"><a href="#cb17-6" tabindex="-1"></a><span class="im">import</span> torch.nn.functional <span class="im">as</span> F</span>
+<span id="cb17-7"><a href="#cb17-7" tabindex="-1"></a><span class="im">import</span> matplotlib.pyplot <span class="im">as</span> plt</span>
+<span id="cb17-8"><a href="#cb17-8" tabindex="-1"></a><span class="im">import</span> numpy <span class="im">as</span> np</span>
+<span id="cb17-9"><a href="#cb17-9" tabindex="-1"></a></span>
+<span id="cb17-10"><a href="#cb17-10" tabindex="-1"></a><span class="co"># Load Fashion MNIST dataset</span></span>
+<span id="cb17-11"><a href="#cb17-11" tabindex="-1"></a>(train_images, train_labels), (test_images, test_labels) <span class="op">=</span> fashion_mnist.load_data()</span>
+<span id="cb17-12"><a href="#cb17-12" tabindex="-1"></a></span>
+<span id="cb17-13"><a href="#cb17-13" tabindex="-1"></a><span class="co"># Define classes for simplicity</span></span>
+<span id="cb17-14"><a href="#cb17-14" tabindex="-1"></a>class_names <span class="op">=</span> [<span class="st">'T-shirt/top'</span>, <span class="st">'Trouser'</span>, <span class="st">'Pullover'</span>, <span class="st">'Dress'</span>, <span class="st">'Coat'</span>,</span>
+<span id="cb17-15"><a href="#cb17-15" tabindex="-1"></a>               <span class="st">'Sandal'</span>, <span class="st">'Shirt'</span>, <span class="st">'Sneaker'</span>, <span class="st">'Bag'</span>, <span class="st">'Ankle boot'</span>]</span>
+<span id="cb17-16"><a href="#cb17-16" tabindex="-1"></a></span>
+<span id="cb17-17"><a href="#cb17-17" tabindex="-1"></a><span class="co"># Prepare OOD data - Sandals (5)</span></span>
+<span id="cb17-18"><a href="#cb17-18" tabindex="-1"></a>ood_data <span class="op">=</span> test_images[test_labels <span class="op">==</span> <span class="dv">5</span>]</span>
+<span id="cb17-19"><a href="#cb17-19" tabindex="-1"></a>ood_labels <span class="op">=</span> test_labels[test_labels <span class="op">==</span> <span class="dv">5</span>]</span>
+<span id="cb17-20"><a href="#cb17-20" tabindex="-1"></a><span class="bu">print</span>(<span class="ss">f'ood_data.shape=</span><span class="sc">{</span>ood_data<span class="sc">.</span>shape<span class="sc">}</span><span class="ss">'</span>)</span>
+<span id="cb17-21"><a href="#cb17-21" tabindex="-1"></a></span>
+<span id="cb17-22"><a href="#cb17-22" tabindex="-1"></a><span class="co"># Filter data for T-shirts (0) and Trousers (1) as in-distribution</span></span>
+<span id="cb17-23"><a href="#cb17-23" tabindex="-1"></a>train_filter <span class="op">=</span> np.isin(train_labels, [<span class="dv">0</span>, <span class="dv">1</span>])</span>
+<span id="cb17-24"><a href="#cb17-24" tabindex="-1"></a>test_filter <span class="op">=</span> np.isin(test_labels, [<span class="dv">0</span>, <span class="dv">1</span>])</span>
+<span id="cb17-25"><a href="#cb17-25" tabindex="-1"></a></span>
+<span id="cb17-26"><a href="#cb17-26" tabindex="-1"></a>train_data <span class="op">=</span> train_images[train_filter]</span>
+<span id="cb17-27"><a href="#cb17-27" tabindex="-1"></a>train_labels <span class="op">=</span> train_labels[train_filter]</span>
+<span id="cb17-28"><a href="#cb17-28" tabindex="-1"></a><span class="bu">print</span>(<span class="ss">f'train_data.shape=</span><span class="sc">{</span>train_data<span class="sc">.</span>shape<span class="sc">}</span><span class="ss">'</span>)</span>
+<span id="cb17-29"><a href="#cb17-29" tabindex="-1"></a></span>
+<span id="cb17-30"><a href="#cb17-30" tabindex="-1"></a>test_data <span class="op">=</span> test_images[test_filter]</span>
+<span id="cb17-31"><a href="#cb17-31" tabindex="-1"></a>test_labels <span class="op">=</span> test_labels[test_filter]</span>
+<span id="cb17-32"><a href="#cb17-32" tabindex="-1"></a><span class="bu">print</span>(<span class="ss">f'test_data.shape=</span><span class="sc">{</span>test_data<span class="sc">.</span>shape<span class="sc">}</span><span class="ss">'</span>)</span>
+<span id="cb17-33"><a href="#cb17-33" tabindex="-1"></a></span>
+<span id="cb17-34"><a href="#cb17-34" tabindex="-1"></a><span class="co"># Transform to Tensor and normalize</span></span>
+<span id="cb17-35"><a href="#cb17-35" tabindex="-1"></a>transform <span class="op">=</span> transforms.Compose([</span>
+<span id="cb17-36"><a href="#cb17-36" tabindex="-1"></a>    transforms.ToTensor(),</span>
+<span id="cb17-37"><a href="#cb17-37" tabindex="-1"></a>    transforms.Normalize((<span class="fl">0.5</span>,), (<span class="fl">0.5</span>,))</span>
+<span id="cb17-38"><a href="#cb17-38" tabindex="-1"></a>])</span>
+<span id="cb17-39"><a href="#cb17-39" tabindex="-1"></a></span>
+<span id="cb17-40"><a href="#cb17-40" tabindex="-1"></a><span class="co"># Convert to PyTorch tensors and normalize</span></span>
+<span id="cb17-41"><a href="#cb17-41" tabindex="-1"></a>train_data_tensor <span class="op">=</span> torch.tensor(train_data, dtype<span class="op">=</span>torch.float32).unsqueeze(<span class="dv">1</span>) <span class="op">/</span> <span class="fl">255.0</span></span>
+<span id="cb17-42"><a href="#cb17-42" tabindex="-1"></a>test_data_tensor <span class="op">=</span> torch.tensor(test_data, dtype<span class="op">=</span>torch.float32).unsqueeze(<span class="dv">1</span>) <span class="op">/</span> <span class="fl">255.0</span></span>
+<span id="cb17-43"><a href="#cb17-43" tabindex="-1"></a>ood_data_tensor <span class="op">=</span> torch.tensor(ood_data, dtype<span class="op">=</span>torch.float32).unsqueeze(<span class="dv">1</span>) <span class="op">/</span> <span class="fl">255.0</span></span>
+<span id="cb17-44"><a href="#cb17-44" tabindex="-1"></a></span>
+<span id="cb17-45"><a href="#cb17-45" tabindex="-1"></a>train_labels_tensor <span class="op">=</span> torch.tensor(train_labels, dtype<span class="op">=</span>torch.<span class="bu">long</span>)</span>
+<span id="cb17-46"><a href="#cb17-46" tabindex="-1"></a>test_labels_tensor <span class="op">=</span> torch.tensor(test_labels, dtype<span class="op">=</span>torch.<span class="bu">long</span>)</span>
+<span id="cb17-47"><a href="#cb17-47" tabindex="-1"></a></span>
+<span id="cb17-48"><a href="#cb17-48" tabindex="-1"></a>train_dataset <span class="op">=</span> torch.utils.data.TensorDataset(train_data_tensor, train_labels_tensor)</span>
+<span id="cb17-49"><a href="#cb17-49" tabindex="-1"></a>test_dataset <span class="op">=</span> torch.utils.data.TensorDataset(test_data_tensor, test_labels_tensor)</span>
+<span id="cb17-50"><a href="#cb17-50" tabindex="-1"></a>ood_dataset <span class="op">=</span> torch.utils.data.TensorDataset(ood_data_tensor, torch.zeros(ood_data_tensor.shape[<span class="dv">0</span>], dtype<span class="op">=</span>torch.<span class="bu">long</span>))</span>
+<span id="cb17-51"><a href="#cb17-51" tabindex="-1"></a></span>
+<span id="cb17-52"><a href="#cb17-52" tabindex="-1"></a>train_loader <span class="op">=</span> torch.utils.data.DataLoader(train_dataset, batch_size<span class="op">=</span><span class="dv">64</span>, shuffle<span class="op">=</span><span class="va">True</span>)</span>
+<span id="cb17-53"><a href="#cb17-53" tabindex="-1"></a>test_loader <span class="op">=</span> torch.utils.data.DataLoader(test_dataset, batch_size<span class="op">=</span><span class="dv">64</span>, shuffle<span class="op">=</span><span class="va">False</span>)</span>
+<span id="cb17-54"><a href="#cb17-54" tabindex="-1"></a>ood_loader <span class="op">=</span> torch.utils.data.DataLoader(ood_dataset, batch_size<span class="op">=</span><span class="dv">64</span>, shuffle<span class="op">=</span><span class="va">False</span>)</span>
+<span id="cb17-55"><a href="#cb17-55" tabindex="-1"></a></span>
+<span id="cb17-56"><a href="#cb17-56" tabindex="-1"></a><span class="co"># Define a simple CNN model</span></span>
+<span id="cb17-57"><a href="#cb17-57" tabindex="-1"></a><span class="kw">class</span> SimpleCNN(nn.Module):</span>
+<span id="cb17-58"><a href="#cb17-58" tabindex="-1"></a>    <span class="kw">def</span> <span class="fu">__init__</span>(<span class="va">self</span>):</span>
+<span id="cb17-59"><a href="#cb17-59" tabindex="-1"></a>        <span class="bu">super</span>(SimpleCNN, <span class="va">self</span>).<span class="fu">__init__</span>()</span>
+<span id="cb17-60"><a href="#cb17-60" tabindex="-1"></a>        <span class="va">self</span>.conv1 <span class="op">=</span> nn.Conv2d(<span class="dv">1</span>, <span class="dv">32</span>, kernel_size<span class="op">=</span><span class="dv">3</span>)</span>
+<span id="cb17-61"><a href="#cb17-61" tabindex="-1"></a>        <span class="va">self</span>.conv2 <span class="op">=</span> nn.Conv2d(<span class="dv">32</span>, <span class="dv">64</span>, kernel_size<span class="op">=</span><span class="dv">3</span>)</span>
+<span id="cb17-62"><a href="#cb17-62" tabindex="-1"></a>        <span class="va">self</span>.fc1 <span class="op">=</span> nn.Linear(<span class="dv">64</span><span class="op">*</span><span class="dv">5</span><span class="op">*</span><span class="dv">5</span>, <span class="dv">128</span>)  <span class="co"># Updated this line</span></span>
+<span id="cb17-63"><a href="#cb17-63" tabindex="-1"></a>        <span class="va">self</span>.fc2 <span class="op">=</span> nn.Linear(<span class="dv">128</span>, <span class="dv">2</span>)</span>
+<span id="cb17-64"><a href="#cb17-64" tabindex="-1"></a></span>
+<span id="cb17-65"><a href="#cb17-65" tabindex="-1"></a>    <span class="kw">def</span> forward(<span class="va">self</span>, x):</span>
+<span id="cb17-66"><a href="#cb17-66" tabindex="-1"></a>        x <span class="op">=</span> F.relu(F.max_pool2d(<span class="va">self</span>.conv1(x), <span class="dv">2</span>))</span>
+<span id="cb17-67"><a href="#cb17-67" tabindex="-1"></a>        x <span class="op">=</span> F.relu(F.max_pool2d(<span class="va">self</span>.conv2(x), <span class="dv">2</span>))</span>
+<span id="cb17-68"><a href="#cb17-68" tabindex="-1"></a>        x <span class="op">=</span> x.view(<span class="op">-</span><span class="dv">1</span>, <span class="dv">64</span><span class="op">*</span><span class="dv">5</span><span class="op">*</span><span class="dv">5</span>)  <span class="co"># Updated this line</span></span>
+<span id="cb17-69"><a href="#cb17-69" tabindex="-1"></a>        x <span class="op">=</span> F.relu(<span class="va">self</span>.fc1(x))</span>
+<span id="cb17-70"><a href="#cb17-70" tabindex="-1"></a>        x <span class="op">=</span> <span class="va">self</span>.fc2(x)</span>
+<span id="cb17-71"><a href="#cb17-71" tabindex="-1"></a>        <span class="cf">return</span> x</span>
+<span id="cb17-72"><a href="#cb17-72" tabindex="-1"></a></span>
+<span id="cb17-73"><a href="#cb17-73" tabindex="-1"></a>device <span class="op">=</span> torch.device(<span class="st">'cuda'</span> <span class="cf">if</span> torch.cuda.is_available() <span class="cf">else</span> <span class="st">'cpu'</span>)</span>
+<span id="cb17-74"><a href="#cb17-74" tabindex="-1"></a>model <span class="op">=</span> SimpleCNN().to(device)</span>
+<span id="cb17-75"><a href="#cb17-75" tabindex="-1"></a>criterion <span class="op">=</span> nn.CrossEntropyLoss()</span>
+<span id="cb17-76"><a href="#cb17-76" tabindex="-1"></a>optimizer <span class="op">=</span> optim.Adam(model.parameters(), lr<span class="op">=</span><span class="fl">0.001</span>)</span>
+<span id="cb17-77"><a href="#cb17-77" tabindex="-1"></a></span>
+<span id="cb17-78"><a href="#cb17-78" tabindex="-1"></a><span class="kw">def</span> train_model(model, train_loader, criterion, optimizer, epochs<span class="op">=</span><span class="dv">5</span>):</span>
+<span id="cb17-79"><a href="#cb17-79" tabindex="-1"></a>    model.train()</span>
+<span id="cb17-80"><a href="#cb17-80" tabindex="-1"></a>    <span class="cf">for</span> epoch <span class="kw">in</span> <span class="bu">range</span>(epochs):</span>
+<span id="cb17-81"><a href="#cb17-81" tabindex="-1"></a>        running_loss <span class="op">=</span> <span class="fl">0.0</span></span>
+<span id="cb17-82"><a href="#cb17-82" tabindex="-1"></a>        <span class="cf">for</span> inputs, labels <span class="kw">in</span> train_loader:</span>
+<span id="cb17-83"><a href="#cb17-83" tabindex="-1"></a>            inputs, labels <span class="op">=</span> inputs.to(device), labels.to(device)</span>
+<span id="cb17-84"><a href="#cb17-84" tabindex="-1"></a>            optimizer.zero_grad()</span>
+<span id="cb17-85"><a href="#cb17-85" tabindex="-1"></a>            outputs <span class="op">=</span> model(inputs)</span>
+<span id="cb17-86"><a href="#cb17-86" tabindex="-1"></a>            loss <span class="op">=</span> criterion(outputs, labels)</span>
+<span id="cb17-87"><a href="#cb17-87" tabindex="-1"></a>            loss.backward()</span>
+<span id="cb17-88"><a href="#cb17-88" tabindex="-1"></a>            optimizer.step()</span>
+<span id="cb17-89"><a href="#cb17-89" tabindex="-1"></a>            running_loss <span class="op">+=</span> loss.item()</span>
+<span id="cb17-90"><a href="#cb17-90" tabindex="-1"></a>        <span class="bu">print</span>(<span class="ss">f'Epoch </span><span class="sc">{</span>epoch<span class="op">+</span><span class="dv">1</span><span class="sc">}</span><span class="ss">, Loss: </span><span class="sc">{</span>running_loss<span class="op">/</span><span class="bu">len</span>(train_loader)<span class="sc">}</span><span class="ss">'</span>)</span>
+<span id="cb17-91"><a href="#cb17-91" tabindex="-1"></a></span>
+<span id="cb17-92"><a href="#cb17-92" tabindex="-1"></a>train_model(model, train_loader, criterion, optimizer)</span></code></pre>
+</div>
+<div class="codewrapper sourceCode" id="cb18">
+<h3 class="code-label">PYTHON<i aria-hidden="true" data-feather="chevron-left"></i><i aria-hidden="true" data-feather="chevron-right"></i>
+</h3>
+<pre class="sourceCode python" tabindex="0"><code class="sourceCode python"><span id="cb18-1"><a href="#cb18-1" tabindex="-1"></a><span class="im">from</span> sklearn.metrics <span class="im">import</span> confusion_matrix, ConfusionMatrixDisplay</span>
+<span id="cb18-2"><a href="#cb18-2" tabindex="-1"></a></span>
+<span id="cb18-3"><a href="#cb18-3" tabindex="-1"></a><span class="co"># Function to plot confusion matrix</span></span>
+<span id="cb18-4"><a href="#cb18-4" tabindex="-1"></a><span class="kw">def</span> plot_confusion_matrix(labels, predictions, title):</span>
+<span id="cb18-5"><a href="#cb18-5" tabindex="-1"></a>    cm <span class="op">=</span> confusion_matrix(labels, predictions, labels<span class="op">=</span>[<span class="dv">0</span>, <span class="dv">1</span>])</span>
+<span id="cb18-6"><a href="#cb18-6" tabindex="-1"></a>    disp <span class="op">=</span> ConfusionMatrixDisplay(confusion_matrix<span class="op">=</span>cm, display_labels<span class="op">=</span>[<span class="st">"T-shirt/top"</span>, <span class="st">"Trouser"</span>])</span>
+<span id="cb18-7"><a href="#cb18-7" tabindex="-1"></a>    disp.plot(cmap<span class="op">=</span>plt.cm.Blues)</span>
+<span id="cb18-8"><a href="#cb18-8" tabindex="-1"></a>    plt.title(title)</span>
+<span id="cb18-9"><a href="#cb18-9" tabindex="-1"></a>    plt.show()</span>
+<span id="cb18-10"><a href="#cb18-10" tabindex="-1"></a></span>
+<span id="cb18-11"><a href="#cb18-11" tabindex="-1"></a><span class="co"># Function to evaluate model on a dataset</span></span>
+<span id="cb18-12"><a href="#cb18-12" tabindex="-1"></a><span class="kw">def</span> evaluate_model(model, dataloader, device):</span>
+<span id="cb18-13"><a href="#cb18-13" tabindex="-1"></a>    model.<span class="bu">eval</span>()</span>
+<span id="cb18-14"><a href="#cb18-14" tabindex="-1"></a>    all_labels <span class="op">=</span> []</span>
+<span id="cb18-15"><a href="#cb18-15" tabindex="-1"></a>    all_predictions <span class="op">=</span> []</span>
+<span id="cb18-16"><a href="#cb18-16" tabindex="-1"></a>    <span class="cf">with</span> torch.no_grad():</span>
+<span id="cb18-17"><a href="#cb18-17" tabindex="-1"></a>        <span class="cf">for</span> inputs, labels <span class="kw">in</span> dataloader:</span>
+<span id="cb18-18"><a href="#cb18-18" tabindex="-1"></a>            inputs, labels <span class="op">=</span> inputs.to(device), labels.to(device)</span>
+<span id="cb18-19"><a href="#cb18-19" tabindex="-1"></a>            outputs <span class="op">=</span> model(inputs)</span>
+<span id="cb18-20"><a href="#cb18-20" tabindex="-1"></a>            _, preds <span class="op">=</span> torch.<span class="bu">max</span>(outputs, <span class="dv">1</span>)</span>
+<span id="cb18-21"><a href="#cb18-21" tabindex="-1"></a>            all_labels.extend(labels.cpu().numpy())</span>
+<span id="cb18-22"><a href="#cb18-22" tabindex="-1"></a>            all_predictions.extend(preds.cpu().numpy())</span>
+<span id="cb18-23"><a href="#cb18-23" tabindex="-1"></a>    <span class="cf">return</span> np.array(all_labels), np.array(all_predictions)</span>
+<span id="cb18-24"><a href="#cb18-24" tabindex="-1"></a></span>
+<span id="cb18-25"><a href="#cb18-25" tabindex="-1"></a><span class="co"># Evaluate on train data</span></span>
+<span id="cb18-26"><a href="#cb18-26" tabindex="-1"></a>train_labels, train_predictions <span class="op">=</span> evaluate_model(model, train_loader, device)</span>
+<span id="cb18-27"><a href="#cb18-27" tabindex="-1"></a>plot_confusion_matrix(train_labels, train_predictions, <span class="st">"Confusion Matrix for Train Data"</span>)</span>
+<span id="cb18-28"><a href="#cb18-28" tabindex="-1"></a></span>
+<span id="cb18-29"><a href="#cb18-29" tabindex="-1"></a><span class="co"># Evaluate on test data</span></span>
+<span id="cb18-30"><a href="#cb18-30" tabindex="-1"></a>test_labels, test_predictions <span class="op">=</span> evaluate_model(model, test_loader, device)</span>
+<span id="cb18-31"><a href="#cb18-31" tabindex="-1"></a>plot_confusion_matrix(test_labels, test_predictions, <span class="st">"Confusion Matrix for Test Data"</span>)</span></code></pre>
+</div>
+<div class="codewrapper sourceCode" id="cb19">
+<h3 class="code-label">PYTHON<i aria-hidden="true" data-feather="chevron-left"></i><i aria-hidden="true" data-feather="chevron-right"></i>
+</h3>
+<pre class="sourceCode python" tabindex="0"><code class="sourceCode python"><span id="cb19-1"><a href="#cb19-1" tabindex="-1"></a><span class="im">from</span> scipy.stats <span class="im">import</span> gaussian_kde</span>
+<span id="cb19-2"><a href="#cb19-2" tabindex="-1"></a><span class="im">from</span> pytorch_ood.detector <span class="im">import</span> EnergyBased</span>
+<span id="cb19-3"><a href="#cb19-3" tabindex="-1"></a><span class="im">from</span> sklearn.metrics <span class="im">import</span> precision_recall_fscore_support, accuracy_score</span>
+<span id="cb19-4"><a href="#cb19-4" tabindex="-1"></a></span>
+<span id="cb19-5"><a href="#cb19-5" tabindex="-1"></a><span class="co"># Compute softmax scores</span></span>
+<span id="cb19-6"><a href="#cb19-6" tabindex="-1"></a><span class="kw">def</span> get_softmax_scores(model, dataloader):</span>
+<span id="cb19-7"><a href="#cb19-7" tabindex="-1"></a>    model.<span class="bu">eval</span>()</span>
+<span id="cb19-8"><a href="#cb19-8" tabindex="-1"></a>    softmax_scores <span class="op">=</span> []</span>
+<span id="cb19-9"><a href="#cb19-9" tabindex="-1"></a>    <span class="cf">with</span> torch.no_grad():</span>
+<span id="cb19-10"><a href="#cb19-10" tabindex="-1"></a>        <span class="cf">for</span> inputs, _ <span class="kw">in</span> dataloader:</span>
+<span id="cb19-11"><a href="#cb19-11" tabindex="-1"></a>            inputs <span class="op">=</span> inputs.to(device)</span>
+<span id="cb19-12"><a href="#cb19-12" tabindex="-1"></a>            outputs <span class="op">=</span> model(inputs)</span>
+<span id="cb19-13"><a href="#cb19-13" tabindex="-1"></a>            softmax <span class="op">=</span> torch.nn.functional.softmax(outputs, dim<span class="op">=</span><span class="dv">1</span>)</span>
+<span id="cb19-14"><a href="#cb19-14" tabindex="-1"></a>            softmax_scores.extend(softmax.cpu().numpy())</span>
+<span id="cb19-15"><a href="#cb19-15" tabindex="-1"></a>    <span class="cf">return</span> np.array(softmax_scores)</span>
+<span id="cb19-16"><a href="#cb19-16" tabindex="-1"></a></span>
+<span id="cb19-17"><a href="#cb19-17" tabindex="-1"></a>id_softmax_scores <span class="op">=</span> get_softmax_scores(model, test_loader)</span>
+<span id="cb19-18"><a href="#cb19-18" tabindex="-1"></a>ood_softmax_scores <span class="op">=</span> get_softmax_scores(model, ood_loader)</span>
+<span id="cb19-19"><a href="#cb19-19" tabindex="-1"></a></span>
+<span id="cb19-20"><a href="#cb19-20" tabindex="-1"></a><span class="co"># Initialize the energy-based OOD detector</span></span>
+<span id="cb19-21"><a href="#cb19-21" tabindex="-1"></a>energy_detector <span class="op">=</span> EnergyBased(model, t<span class="op">=</span><span class="fl">1.0</span>)</span>
+<span id="cb19-22"><a href="#cb19-22" tabindex="-1"></a></span>
+<span id="cb19-23"><a href="#cb19-23" tabindex="-1"></a><span class="co"># Compute energy scores</span></span>
+<span id="cb19-24"><a href="#cb19-24" tabindex="-1"></a><span class="kw">def</span> get_energy_scores(detector, dataloader):</span>
+<span id="cb19-25"><a href="#cb19-25" tabindex="-1"></a>    scores <span class="op">=</span> []</span>
+<span id="cb19-26"><a href="#cb19-26" tabindex="-1"></a>    detector.model.<span class="bu">eval</span>()</span>
+<span id="cb19-27"><a href="#cb19-27" tabindex="-1"></a>    <span class="cf">with</span> torch.no_grad():</span>
+<span id="cb19-28"><a href="#cb19-28" tabindex="-1"></a>        <span class="cf">for</span> inputs, _ <span class="kw">in</span> dataloader:</span>
+<span id="cb19-29"><a href="#cb19-29" tabindex="-1"></a>            inputs <span class="op">=</span> inputs.to(device)</span>
+<span id="cb19-30"><a href="#cb19-30" tabindex="-1"></a>            score <span class="op">=</span> detector.predict(inputs)</span>
+<span id="cb19-31"><a href="#cb19-31" tabindex="-1"></a>            scores.extend(score.cpu().numpy())</span>
+<span id="cb19-32"><a href="#cb19-32" tabindex="-1"></a>    <span class="cf">return</span> np.array(scores)</span>
+<span id="cb19-33"><a href="#cb19-33" tabindex="-1"></a></span>
+<span id="cb19-34"><a href="#cb19-34" tabindex="-1"></a>id_energy_scores <span class="op">=</span> get_energy_scores(energy_detector, test_loader)</span>
+<span id="cb19-35"><a href="#cb19-35" tabindex="-1"></a>ood_energy_scores <span class="op">=</span> get_energy_scores(energy_detector, ood_loader)</span>
+<span id="cb19-36"><a href="#cb19-36" tabindex="-1"></a></span>
+<span id="cb19-37"><a href="#cb19-37" tabindex="-1"></a><span class="im">import</span> matplotlib.pyplot <span class="im">as</span> plt</span>
+<span id="cb19-38"><a href="#cb19-38" tabindex="-1"></a></span>
+<span id="cb19-39"><a href="#cb19-39" tabindex="-1"></a></span>
+<span id="cb19-40"><a href="#cb19-40" tabindex="-1"></a><span class="co"># Plot PSDs</span></span>
+<span id="cb19-41"><a href="#cb19-41" tabindex="-1"></a></span>
+<span id="cb19-42"><a href="#cb19-42" tabindex="-1"></a><span class="co"># Function to plot PSD</span></span>
+<span id="cb19-43"><a href="#cb19-43" tabindex="-1"></a><span class="kw">def</span> plot_psd(id_scores, ood_scores, method_name):</span>
+<span id="cb19-44"><a href="#cb19-44" tabindex="-1"></a>    plt.figure(figsize<span class="op">=</span>(<span class="dv">12</span>, <span class="dv">6</span>))</span>
+<span id="cb19-45"><a href="#cb19-45" tabindex="-1"></a>    alpha <span class="op">=</span> <span class="fl">0.3</span></span>
+<span id="cb19-46"><a href="#cb19-46" tabindex="-1"></a></span>
+<span id="cb19-47"><a href="#cb19-47" tabindex="-1"></a>    <span class="co"># Plot PSD for ID scores</span></span>
+<span id="cb19-48"><a href="#cb19-48" tabindex="-1"></a>    id_density <span class="op">=</span> gaussian_kde(id_scores)</span>
+<span id="cb19-49"><a href="#cb19-49" tabindex="-1"></a>    x_id <span class="op">=</span> np.linspace(id_scores.<span class="bu">min</span>(), id_scores.<span class="bu">max</span>(), <span class="dv">1000</span>)</span>
+<span id="cb19-50"><a href="#cb19-50" tabindex="-1"></a>    plt.plot(x_id, id_density(x_id), label<span class="op">=</span><span class="ss">f'ID (</span><span class="sc">{</span>method_name<span class="sc">}</span><span class="ss">)'</span>, color<span class="op">=</span><span class="st">'blue'</span>, alpha<span class="op">=</span>alpha)</span>
+<span id="cb19-51"><a href="#cb19-51" tabindex="-1"></a></span>
+<span id="cb19-52"><a href="#cb19-52" tabindex="-1"></a>    <span class="co"># Plot PSD for OOD scores</span></span>
+<span id="cb19-53"><a href="#cb19-53" tabindex="-1"></a>    ood_density <span class="op">=</span> gaussian_kde(ood_scores)</span>
+<span id="cb19-54"><a href="#cb19-54" tabindex="-1"></a>    x_ood <span class="op">=</span> np.linspace(ood_scores.<span class="bu">min</span>(), ood_scores.<span class="bu">max</span>(), <span class="dv">1000</span>)</span>
+<span id="cb19-55"><a href="#cb19-55" tabindex="-1"></a>    plt.plot(x_ood, ood_density(x_ood), label<span class="op">=</span><span class="ss">f'OOD (</span><span class="sc">{</span>method_name<span class="sc">}</span><span class="ss">)'</span>, color<span class="op">=</span><span class="st">'red'</span>, alpha<span class="op">=</span>alpha)</span>
+<span id="cb19-56"><a href="#cb19-56" tabindex="-1"></a></span>
+<span id="cb19-57"><a href="#cb19-57" tabindex="-1"></a>    plt.xlabel(<span class="st">'Score'</span>)</span>
+<span id="cb19-58"><a href="#cb19-58" tabindex="-1"></a>    plt.ylabel(<span class="st">'Density'</span>)</span>
+<span id="cb19-59"><a href="#cb19-59" tabindex="-1"></a>    plt.title(<span class="ss">f'Probability Density Distributions for </span><span class="sc">{</span>method_name<span class="sc">}</span><span class="ss"> Scores'</span>)</span>
+<span id="cb19-60"><a href="#cb19-60" tabindex="-1"></a>    plt.legend()</span>
+<span id="cb19-61"><a href="#cb19-61" tabindex="-1"></a>    plt.show()</span>
+<span id="cb19-62"><a href="#cb19-62" tabindex="-1"></a></span>
+<span id="cb19-63"><a href="#cb19-63" tabindex="-1"></a><span class="co"># Plot PSD for softmax scores</span></span>
+<span id="cb19-64"><a href="#cb19-64" tabindex="-1"></a>plot_psd(id_softmax_scores[:, <span class="dv">1</span>], ood_softmax_scores[:, <span class="dv">1</span>], <span class="st">'Softmax'</span>)</span>
+<span id="cb19-65"><a href="#cb19-65" tabindex="-1"></a></span>
+<span id="cb19-66"><a href="#cb19-66" tabindex="-1"></a><span class="co"># Plot PSD for energy scores</span></span>
+<span id="cb19-67"><a href="#cb19-67" tabindex="-1"></a>plot_psd(id_energy_scores, ood_energy_scores, <span class="st">'Energy'</span>)</span>
+<span id="cb19-68"><a href="#cb19-68" tabindex="-1"></a></span></code></pre>
+</div>
+<div class="codewrapper sourceCode" id="cb20">
+<h3 class="code-label">PYTHON<i aria-hidden="true" data-feather="chevron-left"></i><i aria-hidden="true" data-feather="chevron-right"></i>
+</h3>
+<pre class="sourceCode python" tabindex="0"><code class="sourceCode python"><span id="cb20-1"><a href="#cb20-1" tabindex="-1"></a><span class="im">import</span> numpy <span class="im">as</span> np</span>
+<span id="cb20-2"><a href="#cb20-2" tabindex="-1"></a><span class="im">import</span> matplotlib.pyplot <span class="im">as</span> plt</span>
+<span id="cb20-3"><a href="#cb20-3" tabindex="-1"></a><span class="im">from</span> sklearn.metrics <span class="im">import</span> precision_recall_fscore_support, accuracy_score, confusion_matrix, ConfusionMatrixDisplay</span>
+<span id="cb20-4"><a href="#cb20-4" tabindex="-1"></a></span>
+<span id="cb20-5"><a href="#cb20-5" tabindex="-1"></a><span class="co"># Define thresholds to evaluate</span></span>
+<span id="cb20-6"><a href="#cb20-6" tabindex="-1"></a>thresholds <span class="op">=</span> np.linspace(id_energy_scores.<span class="bu">min</span>(), id_energy_scores.<span class="bu">max</span>(), <span class="dv">50</span>)</span>
+<span id="cb20-7"><a href="#cb20-7" tabindex="-1"></a></span>
+<span id="cb20-8"><a href="#cb20-8" tabindex="-1"></a><span class="co"># Store evaluation metrics for each threshold</span></span>
+<span id="cb20-9"><a href="#cb20-9" tabindex="-1"></a>accuracies <span class="op">=</span> []</span>
+<span id="cb20-10"><a href="#cb20-10" tabindex="-1"></a>precisions <span class="op">=</span> []</span>
+<span id="cb20-11"><a href="#cb20-11" tabindex="-1"></a>recalls <span class="op">=</span> []</span>
+<span id="cb20-12"><a href="#cb20-12" tabindex="-1"></a>f1_scores <span class="op">=</span> []</span>
+<span id="cb20-13"><a href="#cb20-13" tabindex="-1"></a></span>
+<span id="cb20-14"><a href="#cb20-14" tabindex="-1"></a><span class="co"># True labels for OOD data (since they are not part of the original labels)</span></span>
+<span id="cb20-15"><a href="#cb20-15" tabindex="-1"></a>ood_true_labels <span class="op">=</span> np.full(<span class="bu">len</span>(ood_energy_scores), <span class="op">-</span><span class="dv">1</span>)</span>
+<span id="cb20-16"><a href="#cb20-16" tabindex="-1"></a></span>
+<span id="cb20-17"><a href="#cb20-17" tabindex="-1"></a><span class="co"># We need the test_labels to be aligned with the ID data</span></span>
+<span id="cb20-18"><a href="#cb20-18" tabindex="-1"></a>id_true_labels <span class="op">=</span> test_labels[:<span class="bu">len</span>(id_energy_scores)]</span>
+<span id="cb20-19"><a href="#cb20-19" tabindex="-1"></a></span>
+<span id="cb20-20"><a href="#cb20-20" tabindex="-1"></a><span class="cf">for</span> threshold <span class="kw">in</span> thresholds:</span>
+<span id="cb20-21"><a href="#cb20-21" tabindex="-1"></a>    <span class="co"># Classify OOD examples based on energy scores</span></span>
+<span id="cb20-22"><a href="#cb20-22" tabindex="-1"></a>    ood_classifications <span class="op">=</span> np.where(ood_energy_scores <span class="op">&gt;=</span> threshold, <span class="op">-</span><span class="dv">1</span>,  <span class="co"># classified as OOD</span></span>
+<span id="cb20-23"><a href="#cb20-23" tabindex="-1"></a>                                   np.where(ood_energy_scores <span class="op">&lt;</span> threshold, <span class="dv">0</span>, <span class="op">-</span><span class="dv">1</span>))  <span class="co"># classified as ID</span></span>
+<span id="cb20-24"><a href="#cb20-24" tabindex="-1"></a></span>
+<span id="cb20-25"><a href="#cb20-25" tabindex="-1"></a>    <span class="co"># Classify ID examples based on energy scores</span></span>
+<span id="cb20-26"><a href="#cb20-26" tabindex="-1"></a>    id_classifications <span class="op">=</span> np.where(id_energy_scores <span class="op">&gt;=</span> threshold, <span class="op">-</span><span class="dv">1</span>,  <span class="co"># classified as OOD</span></span>
+<span id="cb20-27"><a href="#cb20-27" tabindex="-1"></a>                                  np.where(id_energy_scores <span class="op">&lt;</span> threshold, id_true_labels, <span class="op">-</span><span class="dv">1</span>))  <span class="co"># classified as ID</span></span>
+<span id="cb20-28"><a href="#cb20-28" tabindex="-1"></a></span>
+<span id="cb20-29"><a href="#cb20-29" tabindex="-1"></a>    <span class="co"># Combine OOD and ID classifications and true labels</span></span>
+<span id="cb20-30"><a href="#cb20-30" tabindex="-1"></a>    all_predictions <span class="op">=</span> np.concatenate([ood_classifications, id_classifications])</span>
+<span id="cb20-31"><a href="#cb20-31" tabindex="-1"></a>    all_true_labels <span class="op">=</span> np.concatenate([ood_true_labels, id_true_labels])</span>
+<span id="cb20-32"><a href="#cb20-32" tabindex="-1"></a></span>
+<span id="cb20-33"><a href="#cb20-33" tabindex="-1"></a>    <span class="co"># Evaluate metrics</span></span>
+<span id="cb20-34"><a href="#cb20-34" tabindex="-1"></a>    precision, recall, f1, _ <span class="op">=</span> precision_recall_fscore_support(all_true_labels, all_predictions, labels<span class="op">=</span>[<span class="dv">0</span>, <span class="dv">1</span>], average<span class="op">=</span><span class="st">'macro'</span>, zero_division<span class="op">=</span><span class="dv">0</span>)</span>
+<span id="cb20-35"><a href="#cb20-35" tabindex="-1"></a>    accuracy <span class="op">=</span> accuracy_score(all_true_labels, all_predictions)</span>
+<span id="cb20-36"><a href="#cb20-36" tabindex="-1"></a></span>
+<span id="cb20-37"><a href="#cb20-37" tabindex="-1"></a>    accuracies.append(accuracy)</span>
+<span id="cb20-38"><a href="#cb20-38" tabindex="-1"></a>    precisions.append(precision)</span>
+<span id="cb20-39"><a href="#cb20-39" tabindex="-1"></a>    recalls.append(recall)</span>
+<span id="cb20-40"><a href="#cb20-40" tabindex="-1"></a>    f1_scores.append(f1)</span>
+<span id="cb20-41"><a href="#cb20-41" tabindex="-1"></a></span>
+<span id="cb20-42"><a href="#cb20-42" tabindex="-1"></a><span class="co"># Find the best thresholds for each metric</span></span>
+<span id="cb20-43"><a href="#cb20-43" tabindex="-1"></a>best_f1_index <span class="op">=</span> np.argmax(f1_scores)</span>
+<span id="cb20-44"><a href="#cb20-44" tabindex="-1"></a>best_f1_threshold <span class="op">=</span> thresholds[best_f1_index]</span>
+<span id="cb20-45"><a href="#cb20-45" tabindex="-1"></a></span>
+<span id="cb20-46"><a href="#cb20-46" tabindex="-1"></a>best_precision_index <span class="op">=</span> np.argmax(precisions)</span>
+<span id="cb20-47"><a href="#cb20-47" tabindex="-1"></a>best_precision_threshold <span class="op">=</span> thresholds[best_precision_index]</span>
+<span id="cb20-48"><a href="#cb20-48" tabindex="-1"></a></span>
+<span id="cb20-49"><a href="#cb20-49" tabindex="-1"></a>best_recall_index <span class="op">=</span> np.argmax(recalls)</span>
+<span id="cb20-50"><a href="#cb20-50" tabindex="-1"></a>best_recall_threshold <span class="op">=</span> thresholds[best_recall_index]</span>
+<span id="cb20-51"><a href="#cb20-51" tabindex="-1"></a></span>
+<span id="cb20-52"><a href="#cb20-52" tabindex="-1"></a><span class="bu">print</span>(<span class="ss">f"Best F1 threshold: </span><span class="sc">{</span>best_f1_threshold<span class="sc">}</span><span class="ss">, F1 Score: </span><span class="sc">{</span>f1_scores[best_f1_index]<span class="sc">}</span><span class="ss">"</span>)</span>
+<span id="cb20-53"><a href="#cb20-53" tabindex="-1"></a><span class="bu">print</span>(<span class="ss">f"Best Precision threshold: </span><span class="sc">{</span>best_precision_threshold<span class="sc">}</span><span class="ss">, Precision: </span><span class="sc">{</span>precisions[best_precision_index]<span class="sc">}</span><span class="ss">"</span>)</span>
+<span id="cb20-54"><a href="#cb20-54" tabindex="-1"></a><span class="bu">print</span>(<span class="ss">f"Best Recall threshold: </span><span class="sc">{</span>best_recall_threshold<span class="sc">}</span><span class="ss">, Recall: </span><span class="sc">{</span>recalls[best_recall_index]<span class="sc">}</span><span class="ss">"</span>)</span>
+<span id="cb20-55"><a href="#cb20-55" tabindex="-1"></a></span>
+<span id="cb20-56"><a href="#cb20-56" tabindex="-1"></a><span class="co"># Plot metrics as functions of the threshold</span></span>
+<span id="cb20-57"><a href="#cb20-57" tabindex="-1"></a>plt.figure(figsize<span class="op">=</span>(<span class="dv">12</span>, <span class="dv">8</span>))</span>
+<span id="cb20-58"><a href="#cb20-58" tabindex="-1"></a>plt.plot(thresholds, precisions, label<span class="op">=</span><span class="st">'Precision'</span>, color<span class="op">=</span><span class="st">'g'</span>)</span>
+<span id="cb20-59"><a href="#cb20-59" tabindex="-1"></a>plt.plot(thresholds, recalls, label<span class="op">=</span><span class="st">'Recall'</span>, color<span class="op">=</span><span class="st">'b'</span>)</span>
+<span id="cb20-60"><a href="#cb20-60" tabindex="-1"></a>plt.plot(thresholds, f1_scores, label<span class="op">=</span><span class="st">'F1 Score'</span>, color<span class="op">=</span><span class="st">'r'</span>)</span>
+<span id="cb20-61"><a href="#cb20-61" tabindex="-1"></a></span>
+<span id="cb20-62"><a href="#cb20-62" tabindex="-1"></a><span class="co"># Add best threshold indicators</span></span>
+<span id="cb20-63"><a href="#cb20-63" tabindex="-1"></a>plt.axvline(x<span class="op">=</span>best_f1_threshold, color<span class="op">=</span><span class="st">'r'</span>, linestyle<span class="op">=</span><span class="st">'--'</span>, label<span class="op">=</span><span class="ss">f'Best F1 Threshold: </span><span class="sc">{</span>best_f1_threshold<span class="sc">:.2f}</span><span class="ss">'</span>)</span>
+<span id="cb20-64"><a href="#cb20-64" tabindex="-1"></a>plt.axvline(x<span class="op">=</span>best_precision_threshold, color<span class="op">=</span><span class="st">'g'</span>, linestyle<span class="op">=</span><span class="st">'--'</span>, label<span class="op">=</span><span class="ss">f'Best Precision Threshold: </span><span class="sc">{</span>best_precision_threshold<span class="sc">:.2f}</span><span class="ss">'</span>)</span>
+<span id="cb20-65"><a href="#cb20-65" tabindex="-1"></a>plt.axvline(x<span class="op">=</span>best_recall_threshold, color<span class="op">=</span><span class="st">'b'</span>, linestyle<span class="op">=</span><span class="st">'--'</span>, label<span class="op">=</span><span class="ss">f'Best Recall Threshold: </span><span class="sc">{</span>best_recall_threshold<span class="sc">:.2f}</span><span class="ss">'</span>)</span>
+<span id="cb20-66"><a href="#cb20-66" tabindex="-1"></a></span>
+<span id="cb20-67"><a href="#cb20-67" tabindex="-1"></a>plt.xlabel(<span class="st">'Threshold'</span>)</span>
+<span id="cb20-68"><a href="#cb20-68" tabindex="-1"></a>plt.ylabel(<span class="st">'Metric Value'</span>)</span>
+<span id="cb20-69"><a href="#cb20-69" tabindex="-1"></a>plt.title(<span class="st">'Evaluation Metrics as Functions of Threshold (Energy-Based OOD Detection)'</span>)</span>
+<span id="cb20-70"><a href="#cb20-70" tabindex="-1"></a>plt.legend()</span>
+<span id="cb20-71"><a href="#cb20-71" tabindex="-1"></a>plt.show()</span></code></pre>
+</div>
+<div class="codewrapper sourceCode" id="cb21">
+<h3 class="code-label">PYTHON<i aria-hidden="true" data-feather="chevron-left"></i><i aria-hidden="true" data-feather="chevron-right"></i>
+</h3>
+<pre class="sourceCode python" tabindex="0"><code class="sourceCode python"><span id="cb21-1"><a href="#cb21-1" tabindex="-1"></a><span class="im">import</span> numpy <span class="im">as</span> np</span>
+<span id="cb21-2"><a href="#cb21-2" tabindex="-1"></a><span class="im">import</span> matplotlib.pyplot <span class="im">as</span> plt</span>
+<span id="cb21-3"><a href="#cb21-3" tabindex="-1"></a><span class="im">from</span> sklearn.metrics <span class="im">import</span> confusion_matrix, ConfusionMatrixDisplay</span>
+<span id="cb21-4"><a href="#cb21-4" tabindex="-1"></a></span>
+<span id="cb21-5"><a href="#cb21-5" tabindex="-1"></a><span class="co"># Threshold value for the energy score</span></span>
+<span id="cb21-6"><a href="#cb21-6" tabindex="-1"></a>upper_threshold <span class="op">=</span> best_f1_threshold  <span class="co"># Using the best F1 threshold from the previous calculation</span></span>
+<span id="cb21-7"><a href="#cb21-7" tabindex="-1"></a></span>
+<span id="cb21-8"><a href="#cb21-8" tabindex="-1"></a><span class="co"># Classifying OOD examples based on energy scores</span></span>
+<span id="cb21-9"><a href="#cb21-9" tabindex="-1"></a>ood_classifications <span class="op">=</span> np.where(ood_energy_scores <span class="op">&gt;=</span> upper_threshold, <span class="op">-</span><span class="dv">1</span>,  <span class="co"># classified as OOD</span></span>
+<span id="cb21-10"><a href="#cb21-10" tabindex="-1"></a>                               np.where(ood_energy_scores <span class="op">&lt;</span> upper_threshold, <span class="dv">0</span>, <span class="op">-</span><span class="dv">1</span>))  <span class="co"># classified as ID</span></span>
+<span id="cb21-11"><a href="#cb21-11" tabindex="-1"></a></span>
+<span id="cb21-12"><a href="#cb21-12" tabindex="-1"></a><span class="co"># Classifying ID examples based on energy scores</span></span>
+<span id="cb21-13"><a href="#cb21-13" tabindex="-1"></a>id_classifications <span class="op">=</span> np.where(id_energy_scores <span class="op">&gt;=</span> upper_threshold, <span class="op">-</span><span class="dv">1</span>,  <span class="co"># classified as OOD</span></span>
+<span id="cb21-14"><a href="#cb21-14" tabindex="-1"></a>                              np.where(id_energy_scores <span class="op">&lt;</span> upper_threshold, id_true_labels, <span class="op">-</span><span class="dv">1</span>))  <span class="co"># classified as ID</span></span>
+<span id="cb21-15"><a href="#cb21-15" tabindex="-1"></a></span>
+<span id="cb21-16"><a href="#cb21-16" tabindex="-1"></a><span class="co"># Combine OOD and ID classifications and true labels</span></span>
+<span id="cb21-17"><a href="#cb21-17" tabindex="-1"></a>all_predictions <span class="op">=</span> np.concatenate([ood_classifications, id_classifications])</span>
+<span id="cb21-18"><a href="#cb21-18" tabindex="-1"></a>all_true_labels <span class="op">=</span> np.concatenate([ood_true_labels, id_true_labels])</span>
+<span id="cb21-19"><a href="#cb21-19" tabindex="-1"></a></span>
+<span id="cb21-20"><a href="#cb21-20" tabindex="-1"></a><span class="co"># Confusion matrix</span></span>
+<span id="cb21-21"><a href="#cb21-21" tabindex="-1"></a>cm <span class="op">=</span> confusion_matrix(all_true_labels, all_predictions, labels<span class="op">=</span>[<span class="dv">0</span>, <span class="dv">1</span>, <span class="op">-</span><span class="dv">1</span>])</span>
+<span id="cb21-22"><a href="#cb21-22" tabindex="-1"></a></span>
+<span id="cb21-23"><a href="#cb21-23" tabindex="-1"></a><span class="co"># Plotting the confusion matrix</span></span>
+<span id="cb21-24"><a href="#cb21-24" tabindex="-1"></a>disp <span class="op">=</span> ConfusionMatrixDisplay(confusion_matrix<span class="op">=</span>cm, display_labels<span class="op">=</span>[<span class="st">"Shirt"</span>, <span class="st">"Pants"</span>, <span class="st">"OOD"</span>])</span>
+<span id="cb21-25"><a href="#cb21-25" tabindex="-1"></a>disp.plot(cmap<span class="op">=</span>plt.cm.Blues)</span>
+<span id="cb21-26"><a href="#cb21-26" tabindex="-1"></a>plt.title(<span class="st">'Confusion Matrix for OOD and ID Classification (Energy-Based)'</span>)</span>
+<span id="cb21-27"><a href="#cb21-27" tabindex="-1"></a>plt.show()</span></code></pre>
+</div>
+</div>
+</div>
+</div>
+<div class="section level1">
+<h1 id="conclusion">Conclusion<a class="anchor" aria-label="anchor" href="#conclusion"></a>
+</h1>
+<div class="codewrapper sourceCode" id="cb22">
 <h3 class="code-label">PYTHON<i aria-hidden="true" data-feather="chevron-left"></i><i aria-hidden="true" data-feather="chevron-right"></i>
 </h3>
 <pre class="sourceCode python" tabindex="0"><code class="sourceCode python"></code></pre>
 </div>
+<div class="section level2">
+<h2 id="references-and-supplemental-resources">References and supplemental resources<a class="anchor" aria-label="anchor" href="#references-and-supplemental-resources"></a>
+</h2>
+<ul>
+<li><a href="https://www.youtube.com/watch?v=hgLC9_9ZCJI" class="external-link uri">https://www.youtube.com/watch?v=hgLC9_9ZCJI</a></li>
+<li>Generalized Out-of-Distribution Detection: A Survey: <a href="https://arxiv.org/abs/2110.11334" class="external-link uri">https://arxiv.org/abs/2110.11334</a> # Glossary</li>
+<li>ID/OOD: In-distribution, out-of-distribution. Generally, the OOD
+instances can be defined as instances (x, y) sampled from an underlying
+distribution other than the training distribution P(Xtrain, Ytrain),
+where Xtrain and Ytrain are the training corpus and training label set,
+respectively.</li>
+<li>OOD instances with semantic shift: OOD instances with semantic shift
+refer to instances that do not belong to y_train. More specifically,
+instances with semantic shift may come from unknown categories or
+irrelevant tasks.</li>
+<li>OOD instances with covariate shift: OOD instances with non-semantic
+shift refer to the instances that belong to y_train but are sampled from
+a distribution other than x_train, e.g., a different
+domain/corpus/location.</li>
+<li>Closed-world assumption: an assumption that the training and test
+data are sampled from the same distribution. However, training data can
+rarely capture the entire distribution. In real-world scenarios,
+out-of-distribution (OOD) instances, which come from categories that are
+not known to the model, can often be present in inference phases.</li>
+<li>Inference-time OOD: After training, use some kind of scoring
+function to determine if test inputs are OOD or not.</li>
+<li>Output-based OOD: Output-based OOD detection methods leverage the
+model’s output distribution to identify OOD instances. These methods
+typically involve analyzing the softmax scores, confidence scores, or
+other output statistics to detect anomalies.</li>
+</ul>
 <!--
 Place links that you need to refer to multiple times across pages here. Delete
 any links that you are not going to use.
  -->
+</div>
 </div></section><section id="aio-7b-OOD-detection-distance-based"><p>Content from <a href="7b-OOD-detection-distance-based.html">OOD Detection: Distance-Based and Contrastive Learning</a></p>
 <hr>
 <p>Last updated on 2024-07-31 |
diff --git a/instructor/images.html b/instructor/images.html
index a3f73f6d..6f23310d 100644
--- a/instructor/images.html
+++ b/instructor/images.html
@@ -539,7 +539,7 @@ <h3 id="5a-explainable-AI-method-overview-figure-6">Figure 6</h3>
 <hr class="half-width"></section><section id="5c-probes"><h2 class="section-heading"><a href="5c-probes.html">Explainability methods: linear probe</a></h2>
 <hr class="half-width"></section><section id="5d-gradcam"><h2 class="section-heading"><a href="5d-gradcam.html">Explainability methods: GradCAM</a></h2>
 <hr class="half-width"></section><section id="6-confidence-intervals"><h2 class="section-heading"><a href="6-confidence-intervals.html">Estimating model uncertainty</a></h2>
-<hr class="half-width"></section><section id="7a-OOD-detection-output-based"><h2 class="section-heading"><a href="7a-OOD-detection-output-based.html">OOD Detection: Overview, Output-Based MethodsIntroduction to Out-of-Distribution (OOD) Data<a class="anchor" aria-label="anchor" href="#introduction-to-out-of-distribution-ood-data"></a>Example 1: Softmax scores<a class="anchor" aria-label="anchor" href="#example-1-softmax-scores"></a>Example 2: Energy-Based OOD Detection<a class="anchor" aria-label="anchor" href="#example-2-energy-based-ood-detection"></a></a></h2>
+<hr class="half-width"></section><section id="7a-OOD-detection-output-based"><h2 class="section-heading"><a href="7a-OOD-detection-output-based.html">OOD Detection: Overview, Output-Based MethodsIntroduction to Out-of-Distribution (OOD) Data<a class="anchor" aria-label="anchor" href="#introduction-to-out-of-distribution-ood-data"></a>Example 1: Softmax scores<a class="anchor" aria-label="anchor" href="#example-1-softmax-scores"></a>Example 2: Energy-Based OOD Detection<a class="anchor" aria-label="anchor" href="#example-2-energy-based-ood-detection"></a>Conclusion<a class="anchor" aria-label="anchor" href="#conclusion"></a></a></h2>
 <hr class="half-width"></section><section id="7b-OOD-detection-distance-based"><h2 class="section-heading"><a href="7b-OOD-detection-distance-based.html">OOD Detection: Distance-Based and Contrastive LearningExample 3: Distance-Based Methods<a class="anchor" aria-label="anchor" href="#example-3-distance-based-methods"></a>Limitations of Threshold-Based OOD Detection Methods<a class="anchor" aria-label="anchor" href="#limitations-of-threshold-based-ood-detection-methods"></a></a></h2>
 <hr class="half-width"></section><section id="7c-OOD-detection-algo-design"><h2 class="section-heading"><a href="7c-OOD-detection-algo-design.html">OOD Detection: Training-Time RegularizationTraining-time regularization for OOD detection<a class="anchor" aria-label="anchor" href="#training-time-regularization-for-ood-detection"></a></a></h2>
 <hr class="half-width"></section><section id="8-releasing-a-model"><h2 class="section-heading"><a href="8-releasing-a-model.html">Documenting and releasing a model</a></h2>
diff --git a/instructor/index.html b/instructor/index.html
index 3f1bab61..abe90dd7 100644
--- a/instructor/index.html
+++ b/instructor/index.html
@@ -530,8 +530,8 @@ <h3 class="callout-title">Prerequisite<a class="anchor" aria-label="anchor" href
 <td>
 What are out-of-distribution (OOD) data and why is detecting them
 important in machine learning models?<br><br>How do output-based
-methods like softmax, energy-based, and distance-based methods work for
-OOD detection?<br><br>What are the limitations of output-based OOD
+methods like softmax and energy-based methods work for OOD
+detection?<br><br>What are the limitations of output-based OOD
 detection
 methods?<br>::::::::::::::::::::::::::::::::::::::::::::::::::<br>:::::::::::::::::::::::::::::::::::::::
 objectives<br><br>Understand the concept of out-of-distribution data
diff --git a/instructor/instructor-notes.html b/instructor/instructor-notes.html
index 1bc10557..6ea49e70 100644
--- a/instructor/instructor-notes.html
+++ b/instructor/instructor-notes.html
@@ -490,7 +490,7 @@ <h3 id="0-introduction-headingInstructor1">
 <hr class="half-width"></section><section id="5c-probes"><h2 class="section-heading"><a href="5c-probes.html">Explainability methods: linear probe</a></h2>
 <hr class="half-width"></section><section id="5d-gradcam"><h2 class="section-heading"><a href="5d-gradcam.html">Explainability methods: GradCAM</a></h2>
 <hr class="half-width"></section><section id="6-confidence-intervals"><h2 class="section-heading"><a href="6-confidence-intervals.html">Estimating model uncertainty</a></h2>
-<hr class="half-width"></section><section id="7a-OOD-detection-output-based"><h2 class="section-heading"><a href="7a-OOD-detection-output-based.html">OOD Detection: Overview, Output-Based MethodsIntroduction to Out-of-Distribution (OOD) Data<a class="anchor" aria-label="anchor" href="#introduction-to-out-of-distribution-ood-data"></a>Example 1: Softmax scores<a class="anchor" aria-label="anchor" href="#example-1-softmax-scores"></a>Example 2: Energy-Based OOD Detection<a class="anchor" aria-label="anchor" href="#example-2-energy-based-ood-detection"></a></a></h2>
+<hr class="half-width"></section><section id="7a-OOD-detection-output-based"><h2 class="section-heading"><a href="7a-OOD-detection-output-based.html">OOD Detection: Overview, Output-Based MethodsIntroduction to Out-of-Distribution (OOD) Data<a class="anchor" aria-label="anchor" href="#introduction-to-out-of-distribution-ood-data"></a>Example 1: Softmax scores<a class="anchor" aria-label="anchor" href="#example-1-softmax-scores"></a>Example 2: Energy-Based OOD Detection<a class="anchor" aria-label="anchor" href="#example-2-energy-based-ood-detection"></a>Conclusion<a class="anchor" aria-label="anchor" href="#conclusion"></a></a></h2>
 <hr class="half-width"></section><section id="7b-OOD-detection-distance-based"><h2 class="section-heading"><a href="7b-OOD-detection-distance-based.html">OOD Detection: Distance-Based and Contrastive LearningExample 3: Distance-Based Methods<a class="anchor" aria-label="anchor" href="#example-3-distance-based-methods"></a>Limitations of Threshold-Based OOD Detection Methods<a class="anchor" aria-label="anchor" href="#limitations-of-threshold-based-ood-detection-methods"></a></a></h2>
 <hr class="half-width"></section><section id="7c-OOD-detection-algo-design"><h2 class="section-heading"><a href="7c-OOD-detection-algo-design.html">OOD Detection: Training-Time RegularizationTraining-time regularization for OOD detection<a class="anchor" aria-label="anchor" href="#training-time-regularization-for-ood-detection"></a></a></h2>
 <hr class="half-width"></section><section id="8-releasing-a-model"><h2 class="section-heading"><a href="8-releasing-a-model.html">Documenting and releasing a model</a></h2>
diff --git a/instructor/key-points.html b/instructor/key-points.html
index e888c15b..3682e1cb 100644
--- a/instructor/key-points.html
+++ b/instructor/key-points.html
@@ -543,7 +543,7 @@ <h2 class="accordion-header" id="flush-headingTwelve">
 <hr class="half-width">
 <ul>
 <li>TODO</li>
-</ul></section><section id="7a-OOD-detection-output-based"><h2 class="section-heading"><a href="7a-OOD-detection-output-based.html">OOD Detection: Overview, Output-Based MethodsIntroduction to Out-of-Distribution (OOD) Data<a class="anchor" aria-label="anchor" href="#introduction-to-out-of-distribution-ood-data"></a>Example 1: Softmax scores<a class="anchor" aria-label="anchor" href="#example-1-softmax-scores"></a>Example 2: Energy-Based OOD Detection<a class="anchor" aria-label="anchor" href="#example-2-energy-based-ood-detection"></a></a></h2>
+</ul></section><section id="7a-OOD-detection-output-based"><h2 class="section-heading"><a href="7a-OOD-detection-output-based.html">OOD Detection: Overview, Output-Based MethodsIntroduction to Out-of-Distribution (OOD) Data<a class="anchor" aria-label="anchor" href="#introduction-to-out-of-distribution-ood-data"></a>Example 1: Softmax scores<a class="anchor" aria-label="anchor" href="#example-1-softmax-scores"></a>Example 2: Energy-Based OOD Detection<a class="anchor" aria-label="anchor" href="#example-2-energy-based-ood-detection"></a>Conclusion<a class="anchor" aria-label="anchor" href="#conclusion"></a></a></h2>
 <hr class="half-width"></section><section id="7b-OOD-detection-distance-based"><h2 class="section-heading"><a href="7b-OOD-detection-distance-based.html">OOD Detection: Distance-Based and Contrastive LearningExample 3: Distance-Based Methods<a class="anchor" aria-label="anchor" href="#example-3-distance-based-methods"></a>Limitations of Threshold-Based OOD Detection Methods<a class="anchor" aria-label="anchor" href="#limitations-of-threshold-based-ood-detection-methods"></a></a></h2>
 <hr class="half-width"></section><section id="7c-OOD-detection-algo-design"><h2 class="section-heading"><a href="7c-OOD-detection-algo-design.html">OOD Detection: Training-Time RegularizationTraining-time regularization for OOD detection<a class="anchor" aria-label="anchor" href="#training-time-regularization-for-ood-detection"></a></a></h2>
 <hr class="half-width"></section><section id="8-releasing-a-model"><h2 class="section-heading"><a href="8-releasing-a-model.html">Documenting and releasing a model</a></h2>
diff --git a/key-points.html b/key-points.html
index 7067c5bb..29af39e3 100644
--- a/key-points.html
+++ b/key-points.html
@@ -541,7 +541,7 @@ <h2 class="accordion-header" id="flush-headingTwelve">
 <hr class="half-width">
 <ul>
 <li>TODO</li>
-</ul></section><section id="7a-OOD-detection-output-based"><h2 class="section-heading"><a href="7a-OOD-detection-output-based.html">OOD Detection: Overview, Output-Based MethodsIntroduction to Out-of-Distribution (OOD) Data<a class="anchor" aria-label="anchor" href="#introduction-to-out-of-distribution-ood-data"></a>Example 1: Softmax scores<a class="anchor" aria-label="anchor" href="#example-1-softmax-scores"></a>Example 2: Energy-Based OOD Detection<a class="anchor" aria-label="anchor" href="#example-2-energy-based-ood-detection"></a></a></h2>
+</ul></section><section id="7a-OOD-detection-output-based"><h2 class="section-heading"><a href="7a-OOD-detection-output-based.html">OOD Detection: Overview, Output-Based MethodsIntroduction to Out-of-Distribution (OOD) Data<a class="anchor" aria-label="anchor" href="#introduction-to-out-of-distribution-ood-data"></a>Example 1: Softmax scores<a class="anchor" aria-label="anchor" href="#example-1-softmax-scores"></a>Example 2: Energy-Based OOD Detection<a class="anchor" aria-label="anchor" href="#example-2-energy-based-ood-detection"></a>Conclusion<a class="anchor" aria-label="anchor" href="#conclusion"></a></a></h2>
 <hr class="half-width"></section><section id="7b-OOD-detection-distance-based"><h2 class="section-heading"><a href="7b-OOD-detection-distance-based.html">OOD Detection: Distance-Based and Contrastive LearningExample 3: Distance-Based Methods<a class="anchor" aria-label="anchor" href="#example-3-distance-based-methods"></a>Limitations of Threshold-Based OOD Detection Methods<a class="anchor" aria-label="anchor" href="#limitations-of-threshold-based-ood-detection-methods"></a></a></h2>
 <hr class="half-width"></section><section id="7c-OOD-detection-algo-design"><h2 class="section-heading"><a href="7c-OOD-detection-algo-design.html">OOD Detection: Training-Time RegularizationTraining-time regularization for OOD detection<a class="anchor" aria-label="anchor" href="#training-time-regularization-for-ood-detection"></a></a></h2>
 <hr class="half-width"></section><section id="8-releasing-a-model"><h2 class="section-heading"><a href="8-releasing-a-model.html">Documenting and releasing a model</a></h2>
diff --git a/md5sum.txt b/md5sum.txt
index 3c267754..e9e9a42e 100644
--- a/md5sum.txt
+++ b/md5sum.txt
@@ -14,7 +14,7 @@
 "episodes/5c-probes.md" "aaea41bf70b1b424eed4fa0e713d6c31" "site/built/5c-probes.md" "2024-07-03"
 "episodes/5d-gradcam.md" "2672f0a37f77627c78d3118fa6ada584" "site/built/5d-gradcam.md" "2024-07-03"
 "episodes/6-confidence-intervals.md" "e4528f5e9c2146213a13f59647f3a9bf" "site/built/6-confidence-intervals.md" "2024-06-19"
-"episodes/7a-OOD-detection-output-based.md" "f0ae77070f4da84b0f7234b7f9748554" "site/built/7a-OOD-detection-output-based.md" "2024-07-31"
+"episodes/7a-OOD-detection-output-based.md" "1a8869923391b34f5687630fc5bac614" "site/built/7a-OOD-detection-output-based.md" "2024-07-31"
 "episodes/7b-OOD-detection-distance-based.md" "57f0e2fd67a2e51d5af3e62ded49fe05" "site/built/7b-OOD-detection-distance-based.md" "2024-07-31"
 "episodes/7c-OOD-detection-algo-design.md" "32fcee2b30e1f714de7d412bdb943d0f" "site/built/7c-OOD-detection-algo-design.md" "2024-07-31"
 "episodes/8-releasing-a-model.md" "e9b0e3e101a854e0f54b24b6eb3f6729" "site/built/8-releasing-a-model.md" "2024-07-16"
diff --git a/pkgdown.yml b/pkgdown.yml
index 2bc0ad6e..b2c7416a 100644
--- a/pkgdown.yml
+++ b/pkgdown.yml
@@ -2,4 +2,4 @@ pandoc: 3.1.11
 pkgdown: 2.1.0
 pkgdown_sha: ~
 articles: {}
-last_built: 2024-07-31T20:42Z
+last_built: 2024-07-31T22:03Z