From 4ebeb77ef8ae1facf24a2de80554ef1cbce3e124 Mon Sep 17 00:00:00 2001 From: Sergio Quijano Date: Sun, 21 Apr 2024 17:25:28 +0200 Subject: [PATCH] feat #70: computing and displaying metrics after model training --- src/MNIST.py | 214 +++++++++++++++++++++++++-------------------------- 1 file changed, 105 insertions(+), 109 deletions(-) diff --git a/src/MNIST.py b/src/MNIST.py index ae787de..b4c8383 100644 --- a/src/MNIST.py +++ b/src/MNIST.py @@ -17,7 +17,7 @@ import torchvision.transforms as transforms import wandb -from lib import utils +from lib import embedding_to_classifier, metrics, utils @dataclass @@ -527,137 +527,133 @@ def extract_embeddings(dataloader, model): # net.eval() -# TODO -- ADAM -- Run our model evaluation -# # Model evaluation -# # ============================================================================== - - -# # Use the network to perform a retrieval task and compute rank@1 and rank@5 accuracy -# with torch.no_grad(): -# net.set_permute(False) - -# train_rank_at_one = metrics.rank_accuracy( -# k=1, -# data_loader=train_loader_augmented, -# network=net, -# max_examples=len(train_loader_augmented), -# fast_implementation=False, -# ) -# test_rank_at_one = metrics.rank_accuracy( -# k=1, -# data_loader=test_loader, -# network=net, -# max_examples=len(test_loader), -# fast_implementation=False, -# ) -# train_rank_at_five = metrics.rank_accuracy( -# k=5, -# data_loader=train_loader_augmented, -# network=net, -# max_examples=len(train_loader_augmented), -# fast_implementation=False, -# ) -# test_rank_at_five = metrics.rank_accuracy( -# k=5, -# data_loader=test_loader, -# network=net, -# max_examples=len(test_loader), -# fast_implementation=False, -# ) +# Model evaluation +# ============================================================================== -# print(f"Train Rank@1 Accuracy: {train_rank_at_one}") -# print(f"Test Rank@1 Accuracy: {test_rank_at_one}") -# print(f"Train Rank@5 Accuracy: {train_rank_at_five}") -# print(f"Test Rank@5 Accuracy: {test_rank_at_five}") - -# # Put this info in wandb -# wandb.log( -# { -# "Final Train Rank@1 Accuracy": train_rank_at_one, -# "Final Test Rank@1 Accuracy": test_rank_at_one, -# "Final Train Rank@5 Accuracy": train_rank_at_five, -# "Final Test Rank@5 Accuracy": test_rank_at_five, -# } -# ) +# Use the network to perform a retrieval task and compute rank@1 and rank@5 accuracy +with torch.no_grad(): + net.set_permute(False) + + train_rank_at_one = metrics.rank_accuracy( + k=1, + data_loader=online_train_loader, + network=net, + max_examples=len(online_train_loader), + fast_implementation=False, + ) + test_rank_at_one = metrics.rank_accuracy( + k=1, + data_loader=online_test_loader, + network=net, + max_examples=len(online_test_loader), + fast_implementation=False, + ) + train_rank_at_five = metrics.rank_accuracy( + k=5, + data_loader=online_train_loader, + network=net, + max_examples=len(online_train_loader), + fast_implementation=False, + ) + test_rank_at_five = metrics.rank_accuracy( + k=5, + data_loader=online_test_loader, + network=net, + max_examples=len(online_test_loader), + fast_implementation=False, + ) + + print("=> 📈 Final Metrics") + print(f"Train Rank@1 Accuracy: {train_rank_at_one}") + print(f"Test Rank@1 Accuracy: {test_rank_at_one}") + print(f"Train Rank@5 Accuracy: {train_rank_at_five}") + print(f"Test Rank@5 Accuracy: {test_rank_at_five}") + print("") -# net.set_permute(True) + # Put this info in wandb + wandb.log( + { + "Final Train Rank@1 Accuracy": train_rank_at_one, + "Final Test Rank@1 Accuracy": test_rank_at_one, + "Final Train Rank@5 Accuracy": train_rank_at_five, + "Final Test Rank@5 Accuracy": test_rank_at_five, + } + ) + net.set_permute(True) -# # Compute the the *silhouette* metric for the produced embedding, on -# # train, validation and test set: -# with torch.no_grad(): -# net.set_permute(False) -# # Try to clean memory, because we can easily run out of memory -# # This provoke the notebook to crash, and all in-memory objects to be lost -# try_to_clean_memory() +# Compute the the *silhouette* metric for the produced embedding, on +# train, validation and test set: +with torch.no_grad(): + net.set_permute(False) -# train_silh = metrics.silhouette(train_loader_augmented, net) -# print(f"Silhouette in training loader: {train_silh}") + # Try to clean memory, because we can easily run out of memory + # This provoke the notebook to crash, and all in-memory objects to be lost + try_to_clean_memory() -# validation_silh = metrics.silhouette(validation_loader_augmented, net) -# print(f"Silhouette in validation loader: {validation_silh}") + print("=> 📈 Silhouette metrics") + train_silh = metrics.silhouette(online_train_loader, net) + print(f"Silhouette in training loader: {train_silh}") -# test_silh = metrics.silhouette(test_loader, net) -# print(f"Silhouette in test loader: {test_silh}") + test_silh = metrics.silhouette(online_test_loader, net) + print(f"Silhouette in test loader: {test_silh}") + print("") -# # Put this info in wandb -# wandb.log( -# { -# "Final Training silh": train_silh, -# "Final Validation silh": validation_silh, -# "Final Test silh": test_silh, -# } -# ) + # Put this info in wandb + wandb.log( + { + "Final Training silh": train_silh, + "Final Test silh": test_silh, + } + ) -# net.set_permute(True) + net.set_permute(True) -# # Show the "criterion" metric on test set -# with torch.no_grad(): -# net.set_permute(False) +# Now take the classifier from the embedding and use it to compute some classification metrics: +with torch.no_grad(): + # Try to clean memory, because we can easily run out of memory + # This provoke the notebook to crash, and all in-memory objects to be lost + try_to_clean_memory() -# core.test_model_online(net, test_loader, parameters["criterion"], online=True) + # With hopefully enough memory, try to convert the embedding to a classificator + number_neigbours = 3 + classifier = embedding_to_classifier.EmbeddingToClassifier( + net, + k=number_neigbours, + data_loader=online_train_loader, + embedding_dimension=2, + ) -# net.set_permute(True) +# See how it works on a small test set +with torch.no_grad(): + net.set_permute(False) + # Show only `max_iterations` classifications + counter = 0 + max_iterations = len(test_dataset) -# # Now take the classifier from the embedding and use it to compute some classification metrics: -# with torch.no_grad(): -# # Try to clean memory, because we can easily run out of memory -# # This provoke the notebook to crash, and all in-memory objects to be lost -# try_to_clean_memory() - -# # With hopefully enough memory, try to convert the embedding to a classificator -# classifier = EmbeddingToClassifier( -# net, -# k=GLOBALS["NUMBER_NEIGHBOURS"], -# data_loader=train_loader_augmented, -# embedding_dimension=GLOBALS["EMBEDDING_DIMENSION"], -# ) + correct = 0 -# # See how it works on a small test set -# with torch.no_grad(): -# net.set_permute(False) + for img, img_class in test_dataset: + predicted_class = classifier.predict(img) -# # Show only `max_iterations` classifications -# counter = 0 -# max_iterations = 20 + if img_class == predicted_class[0]: + correct += 1 -# for img, img_class in test_dataset: -# predicted_class = classifier.predict(img) -# print( -# f"True label: {img_class}, predicted label: {predicted_class[0]}, correct: {img_class == predicted_class[0]}" -# ) + counter += 1 + if counter == max_iterations: + break -# counter += 1 -# if counter == max_iterations: -# break + accuracy = correct / max_iterations + print(f"=> 📈 Metrics on {max_iterations} test images") + print(f"Accuracy: {(accuracy * 100):.3f}% ") -# net.set_permute(True) + net.set_permute(True) -# TODO -- ADAM -- run our plot of the embedding +# TODO -- ADAM -- run our plot of the embedding # # Plot of the embedding # # ============================================================================== # #